Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package rshim for openSUSE:Factory checked in at 2021-03-12 13:33:56 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/rshim (Old) and /work/SRC/openSUSE:Factory/.rshim.new.2401 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "rshim" Fri Mar 12 13:33:56 2021 rev:6 rq:878519 version:2.0.5.10.0 Changes: -------- --- /work/SRC/openSUSE:Factory/rshim/rshim.changes 2021-01-26 14:50:02.723701785 +0100 +++ /work/SRC/openSUSE:Factory/.rshim.new.2401/rshim.changes 2021-03-12 13:34:04.822389848 +0100 @@ -1,0 +2,9 @@ +Thu Mar 11 16:05:12 UTC 2021 - Matthias Brugger <mbrug...@suse.com> + +- update to 2.0.5.10 + * fix possible tmfifo data loss in PCIe interface + * fix USB deadlock + * reduce CPU load when no rshim device present + * add PCIe hotplug support + +------------------------------------------------------------------- Old: ---- rshim-2.0.5.8.3.tar New: ---- rshim-2.0.5.10.0.tar ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ rshim.spec ++++++ --- /var/tmp/diff_new_pack.lD0b3b/_old 2021-03-12 13:34:05.298390516 +0100 +++ /var/tmp/diff_new_pack.lD0b3b/_new 2021-03-12 13:34:05.302390522 +0100 @@ -18,7 +18,7 @@ Name: rshim -Version: 2.0.5.8.3 +Version: 2.0.5.10.0 Release: 0 Summary: User-space driver for Mellanox BlueField SoC License: GPL-2.0-only ++++++ _service ++++++ --- /var/tmp/diff_new_pack.lD0b3b/_old 2021-03-12 13:34:05.334390566 +0100 +++ /var/tmp/diff_new_pack.lD0b3b/_new 2021-03-12 13:34:05.334390566 +0100 @@ -6,9 +6,9 @@ <param name="scm">git</param> <param name="url">https://github.com/Mellanox/rshim-user-space.git</param> <param name="filename">rshim</param> - <param name="versionrewrite-pattern">rshim-(.*)-(.*)</param> + <param name="versionrewrite-pattern">rshim-(.*)-(.*)</param> <param name="versionrewrite-replacement">\1.\2</param> - <param name="versionformat">@PARENT_TAG@.@TAG_OFFSET@</param> + <param name="versionformat">@PARENT_TAG@.@TAG_OFFSET@</param> </service> <service mode="disabled" name="set_version" /> </services> ++++++ rshim-2.0.5.8.3.tar -> rshim-2.0.5.10.0.tar ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rshim-2.0.5.8.3/debian/changelog new/rshim-2.0.5.10.0/debian/changelog --- old/rshim-2.0.5.8.3/debian/changelog 2021-01-23 01:33:56.000000000 +0100 +++ new/rshim-2.0.5.10.0/debian/changelog 2021-03-10 20:17:18.000000000 +0100 @@ -1,3 +1,18 @@ +rshim (2.0.5-10) UNRELEASED; urgency=low + + * PCIe hotplug support + * Reduce CPU utilization when there is no rshim device + + -- Liming Sun <limi...@nvidia.com> Wed, 10 Mar 2021 14:11:21 -0400 + +rshim (2.0.5-9) UNRELEASED; urgency=low + + * Fix potential tmfifo data loss + * Add workaround checking for Bluefield-2 REV-0 + * Fix network traffic stop issue when Tx buffer full + + -- Liming Sun <limi...@nvidia.com> Wed, 27 Jan 2021 10:17:20 -0400 + rshim (2.0.5-8) UNRELEASED; urgency=low * Don't allow any register access when DROP_MODE is set diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rshim-2.0.5.8.3/rhel/rshim.spec.in new/rshim-2.0.5.10.0/rhel/rshim.spec.in --- old/rshim-2.0.5.8.3/rhel/rshim.spec.in 2021-01-23 01:33:56.000000000 +0100 +++ new/rshim-2.0.5.10.0/rhel/rshim.spec.in 2021-03-10 20:17:18.000000000 +0100 @@ -4,7 +4,7 @@ Name: rshim Version: @VERSION@ -Release: 8%{?dist} +Release: 10%{?dist} Summary: User-space driver for Mellanox BlueField SoC License: GPLv2 @@ -54,6 +54,15 @@ %{_mandir}/man8/rshim.8.gz %changelog +* Wed Mar 10 2021 Liming Sun <limi...@nvidia.com> - 2.0.5-10 +- PCIe hotplug support +- Reduce CPU utilization when there is no rshim device + +* Wed Jan 27 2021 Liming Sun <limi...@nvidia.com> - 2.0.5-9 +- Fix potential tmfifo data loss +- Add workaround checking for Bluefield-2 REV-0 +- Fix network traffic stop issue when Tx buffer full + * Fri Dec 11 2020 Liming Sun <limi...@nvidia.com> - 2.0.5-8 - Don't allow any register access when DROP_MODE is set - Avoid potential race in rshim_fifo_read diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rshim-2.0.5.8.3/rshim.spec.in new/rshim-2.0.5.10.0/rshim.spec.in --- old/rshim-2.0.5.8.3/rshim.spec.in 2021-01-23 01:33:56.000000000 +0100 +++ new/rshim-2.0.5.10.0/rshim.spec.in 2021-03-10 20:17:18.000000000 +0100 @@ -4,7 +4,7 @@ Name: rshim Version: @VERSION@ -Release: 8%{?dist} +Release: 10%{?dist} Summary: User-space driver for Mellanox BlueField SoC License: GPLv2 @@ -95,6 +95,15 @@ %{_mandir}/man8/bfb-install.8.gz %changelog +* Wed Mar 10 2021 Liming Sun <limi...@nvidia.com> - 2.0.5-10 +- PCIe hotplug support +- Reduce CPU utilization when there is no rshim device + +* Wed Jan 27 2021 Liming Sun <limi...@nvidia.com> - 2.0.5-9 +- Fix potential tmfifo data loss +- Add workaround checking for Bluefield-2 REV-0 +- Fix network traffic stop issue when Tx buffer full + * Fri Dec 11 2020 Liming Sun <limi...@nvidia.com> - 2.0.5-8 - Don't allow any register access when DROP_MODE is set - Avoid potential race in rshim_fifo_read diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rshim-2.0.5.8.3/src/rshim.c new/rshim-2.0.5.10.0/src/rshim.c --- old/rshim-2.0.5.8.3/src/rshim.c 2021-01-23 01:33:56.000000000 +0100 +++ new/rshim-2.0.5.10.0/src/rshim.c 2021-03-10 20:17:18.000000000 +0100 @@ -28,10 +28,14 @@ /* RShim timer interval in milliseconds. */ #define RSHIM_TIMER_INTERVAL 1 +#define RSHIM_TIMER_INTERVAL_SLOW 100 /* Cycles to poll the network initialization before timeout. */ #define RSHIM_NET_INIT_DELAY (60000 / RSHIM_TIMER_INTERVAL) +/* Reserve some space to indicate full. */ +#define RSHIM_FIFO_SPACE_RESERV 3 + /* Keepalive period in milliseconds. */ static int rshim_keepalive_period = 300; @@ -184,11 +188,19 @@ char *rshim_dev_names[RSHIM_MAX_DEV]; char *rshim_blocked_dev_names[RSHIM_MAX_DEV]; +/* Bitmask of the used rshim device id. */ +#if RSHIM_MAX_DEV > 64 +#error Need to fix the size of rshim_dev_bitmask. +#endif +uint64_t rshim_dev_bitmask; + bool rshim_no_net = false; int rshim_log_level = LOG_NOTICE; bool rshim_daemon_mode = true; volatile bool rshim_run = true; +static uint32_t rshim_timer_interval = RSHIM_TIMER_INTERVAL; + /* Global lock / unlock. */ void rshim_lock(void) @@ -369,7 +381,7 @@ RSHIM_ERR("read_rshim error %d\n", rc); break; } - avail = max_size - (int)(reg & size_mask) - 8; + avail = max_size - (int)(reg & size_mask) - RSHIM_FIFO_SPACE_RESERV; if (avail > 0) break; @@ -878,7 +890,8 @@ RSHIM_ERR("read_rshim error %d\n", ret); return ret; } - avail = max_size - (int)(word & RSH_TM_HOST_TO_TILE_STS__COUNT_MASK) - 1; + avail = max_size - (int)(word & RSH_TM_HOST_TO_TILE_STS__COUNT_MASK) - + RSHIM_FIFO_SPACE_RESERV; return avail; } @@ -1273,15 +1286,20 @@ static void rshim_fifo_output(rshim_backend_t *bd) { - int writesize, write_buf_next = 0; - int write_avail = WRITE_BUF_SIZE - write_buf_next; + int writesize, write_buf_next = 0, write_avail; int numchan = TMFIFO_MAX_CHAN; - int chan, chan_offset; + int chan, chan_offset, fifo_avail; /* If we're already writing, we have nowhere to put data. */ if (bd->spin_flags & RSH_SFLG_WRITING) return; + if (bd->has_reprobe) + fifo_avail = WRITE_BUF_SIZE; + else + fifo_avail = rshim_fifo_tx_avail(bd) * sizeof(uint64_t); + write_avail = fifo_avail - write_buf_next; + if (!bd->write_buf_pkt_rem) { /* Send control messages. */ writesize = rshim_fifo_ctrl_tx(bd); @@ -1292,7 +1310,8 @@ } /* Walk through all the channels, sending as much data as possible. */ - for (chan_offset = 0; chan_offset < numchan; chan_offset++) { + for (chan_offset = 0; chan_offset < numchan && write_avail > 0; + chan_offset++) { /* * Pick the current channel if not done, otherwise round-robin * to the next channel. @@ -1390,7 +1409,7 @@ /* Add padding at the end. */ if (bd->write_buf_pkt_rem == 0) write_buf_next = (write_buf_next + 7) & -8; - write_avail = WRITE_BUF_SIZE - write_buf_next; + write_avail = fifo_avail - write_buf_next; pthread_cond_broadcast(&bd->write_fifo[chan].operable); RSHIM_DBG("fifo_output: woke up writable chan %d\n", chan); @@ -2279,6 +2298,8 @@ } #endif + rshim_dev_bitmask |= (1ULL << index); + return 0; } @@ -2289,6 +2310,8 @@ if (!bd->registered) return; + rshim_dev_bitmask &= ~(1ULL << bd->index); + #ifdef HAVE_RSHIM_FUSE rshim_fuse_del(bd); #endif @@ -2373,6 +2396,18 @@ rshim_unlock(); } +static void rshim_set_timer(int timer_fd, int interval) +{ + struct itimerspec ts; + + ts.it_interval.tv_sec = 0; + ts.it_interval.tv_nsec = interval * 1000000; + ts.it_value.tv_sec = 0; + ts.it_value.tv_nsec = ts.it_interval.tv_nsec; + rshim_timer_interval = interval; + timerfd_settime(timer_fd, 0, &ts, NULL); +} + static void rshim_main(int argc, char *argv[]) { int i, fd, num, rc, epoll_fd, timer_fd, index; @@ -2384,7 +2419,6 @@ #endif struct epoll_event events[MAXEVENTS]; struct epoll_event event; - struct itimerspec ts; rshim_backend_t *bd; time_t t0, t1; uint8_t tmp; @@ -2431,17 +2465,13 @@ exit(-1); } - /* Add timer fd. */ + /* Add periodic timer. */ timer_fd = timerfd_create(CLOCK_MONOTONIC, 0); if (timer_fd == -1) { fprintf(stderr, "timerfd_create failed: %m\n"); exit(1); } - ts.it_interval.tv_sec = 0; - ts.it_interval.tv_nsec = RSHIM_TIMER_INTERVAL * 1000000; - ts.it_value.tv_sec = 0; - ts.it_value.tv_nsec = ts.it_interval.tv_nsec; - timerfd_settime(timer_fd, 0, &ts, NULL); + rshim_set_timer(timer_fd, RSHIM_TIMER_INTERVAL); event.data.fd = timer_fd; event.events = EPOLLIN | EPOLLOUT; rc = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, timer_fd, &event); @@ -2542,6 +2572,14 @@ rshim_pcie_lf_init_done = true; } } + + /* Use slower timer if no rshim devices are found. */ + if (rshim_dev_bitmask) { + if (rshim_timer_interval == RSHIM_TIMER_INTERVAL_SLOW) + rshim_set_timer(timer_fd, RSHIM_TIMER_INTERVAL); + } else if (rshim_timer_interval == RSHIM_TIMER_INTERVAL) { + rshim_set_timer(timer_fd, RSHIM_TIMER_INTERVAL_SLOW); + } } rshim_stop(); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rshim-2.0.5.8.3/src/rshim.h new/rshim-2.0.5.10.0/src/rshim.h --- old/rshim-2.0.5.8.3/src/rshim.h 2021-01-23 01:33:56.000000000 +0100 +++ new/rshim-2.0.5.10.0/src/rshim.h 2021-03-10 20:17:18.000000000 +0100 @@ -151,6 +151,17 @@ #define RSHIM_CHANNEL RSH_MMIO_ADDRESS_SPACE__CHANNEL_VAL_RSHIM #define MMC_CHANNEL RSH_MMIO_ADDRESS_SPACE__CHANNEL_VAL_MMC #define YU_CHANNEL RSH_MMIO_ADDRESS_SPACE__CHANNEL_VAL_TYU +#define UART0_CHANNEL RSH_MMIO_ADDRESS_SPACE__CHANNEL_VAL_UART0 +#define UART1_CHANNEL RSH_MMIO_ADDRESS_SPACE__CHANNEL_VAL_UART1 +#define DIAGUART_CHANNEL RSH_MMIO_ADDRESS_SPACE__CHANNEL_VAL_DIAG_UART +#define OOB_CHANNEL RSH_MMIO_ADDRESS_SPACE__CHANNEL_VAL_TYU_EXT1 +#define TIMER_ARM_CHANNEL RSH_MMIO_ADDRESS_SPACE__CHANNEL_VAL_TIMER +#define RSH_HUB_CHANNEL RSH_MMIO_ADDRESS_SPACE__CHANNEL_VAL_USB +#define TIMER_EXT_CHANNEL RSH_MMIO_ADDRESS_SPACE__CHANNEL_VAL_TIMER_EXT +#define WDOG0_CHANNEL RSH_MMIO_ADDRESS_SPACE__CHANNEL_VAL_WDOG0 +#define WDOG1_CHANNEL RSH_MMIO_ADDRESS_SPACE__CHANNEL_VAL_WDOG1 +#define GIC_CHANNEL RSH_MMIO_ADDRESS_SPACE__CHANNEL_VAL_TYU_EXT2 +#define MCH_CORE_CHANNEL RSH_MMIO_ADDRESS_SPACE__CHANNEL_VAL_TYU_EXT3 /* Base RShim Address */ #define RSH_BASE_ADDR 0x80000000 @@ -203,6 +214,7 @@ /* Bluefield Version. */ #define RSHIM_BLUEFIELD_1 1 #define RSHIM_BLUEFIELD_2 2 +#define RSHIM_BLUEFIELD_3 3 /* Bluefield Revision ID. */ #define BLUEFIELD_REV0 0 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rshim-2.0.5.8.3/src/rshim_pcie.c new/rshim-2.0.5.10.0/src/rshim_pcie.c --- old/rshim-2.0.5.8.3/src/rshim_pcie.c 2021-01-23 01:33:56.000000000 +0100 +++ new/rshim-2.0.5.10.0/src/rshim_pcie.c 2021-03-10 20:17:18.000000000 +0100 @@ -324,10 +324,49 @@ static int rshim_pcie_enable_device(rshim_backend_t *bd, bool enable) { rshim_pcie_t *dev = container_of(bd, rshim_pcie_t, bd); + struct pci_dev *pci_dev = dev->pci_dev; int rc = 0; - if (dev->pci_dev) - rc = rshim_pcie_enable(dev->pci_dev, enable); + if (!pci_dev) + return -ENODEV; + + rc = rshim_pcie_enable(pci_dev, enable); + +#ifdef __linux__ + /* Remap resource0 in case it's changed during pcie hotplug. */ + if (bd) { + rshim_pcie_t *dev = container_of(bd, rshim_pcie_t, bd); + char path[256]; + + if (dev->pci_fd) { + close(dev->pci_fd); + dev->pci_fd = -1; + } + + if (dev->rshim_regs) { + munmap((void *)dev->rshim_regs, PCI_RSHIM_WINDOW_OFFSET); + dev->rshim_regs = NULL; + } + + snprintf(path, sizeof(path), "%s/%04x:%02x:%02x.%1u/resource0", + SYS_BUS_PCI, pci_dev->domain, pci_dev->bus, + pci_dev->dev, pci_dev->func); + + dev->pci_fd = open(path, O_RDWR | O_SYNC); + if (dev->pci_fd < 0) { + RSHIM_ERR("Failed to open %s\n", path); + return -ENODEV; + } + dev->rshim_regs = mmap(NULL, PCI_RSHIM_WINDOW_SIZE, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_LOCKED, dev->pci_fd, + PCI_RSHIM_WINDOW_OFFSET); + if (dev->rshim_regs == MAP_FAILED) { + RSHIM_ERR("Failed to map RShim registers\n"); + return -ENOMEM; + } + } +#endif return rc; } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rshim-2.0.5.8.3/src/rshim_usb.c new/rshim-2.0.5.10.0/src/rshim_usb.c --- old/rshim-2.0.5.8.3/src/rshim_usb.c 2021-01-23 01:33:56.000000000 +0100 +++ new/rshim-2.0.5.10.0/src/rshim_usb.c 2021-03-10 20:17:18.000000000 +0100 @@ -16,6 +16,7 @@ #define USB_TILERA_VENDOR_ID 0x22dc /* Tilera Corporation */ #define USB_BLUEFIELD_1_PRODUCT_ID 0x0004 /* Mellanox Bluefield-1 */ #define USB_BLUEFIELD_2_PRODUCT_ID 0x0214 /* Mellanox Bluefield-2 */ +#define USB_BLUEFIELD_3_PRODUCT_ID 0x0314 /* Mellanox Bluefield-3 */ #define READ_RETRIES 5 #define WRITE_RETRIES 5 @@ -59,7 +60,8 @@ static int rshim_usb_product_ids[] = { USB_BLUEFIELD_1_PRODUCT_ID, - USB_BLUEFIELD_2_PRODUCT_ID + USB_BLUEFIELD_2_PRODUCT_ID, + USB_BLUEFIELD_3_PRODUCT_ID }; static void rshim_usb_delete(rshim_backend_t *bd) @@ -75,23 +77,97 @@ free(dev); } +struct rshim_usb_addr { + uint16_t wvalue; + uint16_t windex; +}; + +struct rshim_usb_addr bf3_wvalue_widx_pair_map[] = { + [RSHIM_CHANNEL] = { + .wvalue = 0x0400, + .windex = 0x0000, + }, + [UART0_CHANNEL] = { + .wvalue = 0x0400, + .windex = 0x4000, + }, + [UART1_CHANNEL] = { + .wvalue = 0x0400, + .windex = 0x5000, + }, + [DIAGUART_CHANNEL] = { + .wvalue = 0x0400, + .windex = 0x6000, + }, + [OOB_CHANNEL] = { + .wvalue = 0x0400, + .windex = 0x7000, + }, + [TIMER_ARM_CHANNEL] = { + .wvalue = 0x0400, + .windex = 0x8000, + }, + [RSH_HUB_CHANNEL] = { + .wvalue = 0x0400, + .windex = 0x9000, + }, + [TIMER_EXT_CHANNEL] = { + .wvalue = 0x0400, + .windex = 0xa000, + }, + [WDOG0_CHANNEL] = { + .wvalue = 0x0402, + .windex = 0x0000, + }, + [WDOG1_CHANNEL] = { + .wvalue = 0x0404, + .windex = 0x0000, + }, + [GIC_CHANNEL] = { + .wvalue = 0x0440, + .windex = 0x0000, + }, + [MCH_CORE_CHANNEL] = { + .wvalue = 0x0480, + .windex = 0x0000, + }, +}; + +static struct rshim_usb_addr get_wvalue_windex(int chan, int addr, uint16_t ver_id) +{ + struct rshim_usb_addr rsh_usb_addr; + + if (ver_id == RSHIM_BLUEFIELD_3) { + rsh_usb_addr.wvalue = bf3_wvalue_widx_pair_map[chan].wvalue; + rsh_usb_addr.windex = bf3_wvalue_widx_pair_map[chan].windex + addr; + } else { + rsh_usb_addr.wvalue = chan; + rsh_usb_addr.windex = addr; + } + + return rsh_usb_addr; +} + /* Rshim read/write routines */ static int rshim_usb_read_rshim(rshim_backend_t *bd, int chan, int addr, uint64_t *result) { rshim_usb_t *dev = container_of(bd, rshim_usb_t, bd); + struct rshim_usb_addr rsh_usb_addr; int rc; if (!bd->has_rshim) return -ENODEV; + rsh_usb_addr = get_wvalue_windex(chan, addr, bd->ver_id); + /* Do a blocking control read and endian conversion. */ rc = libusb_control_transfer(dev->handle, LIBUSB_ENDPOINT_IN | LIBUSB_REQUEST_TYPE_VENDOR | LIBUSB_RECIPIENT_ENDPOINT, - 0, chan, addr, + 0, rsh_usb_addr.wvalue, rsh_usb_addr.windex, (unsigned char *)&dev->ctrl_data, sizeof(dev->ctrl_data), RSHIM_USB_TIMEOUT); @@ -117,18 +193,21 @@ uint64_t value) { rshim_usb_t *dev = container_of(bd, rshim_usb_t, bd); + struct rshim_usb_addr rsh_usb_addr; int rc; if (!bd->has_rshim) return -ENODEV; + rsh_usb_addr = get_wvalue_windex(chan, addr, bd->ver_id); + /* Convert the word to little endian and do blocking control write. */ dev->ctrl_data = htole64(value); rc = libusb_control_transfer(dev->handle, LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR | LIBUSB_RECIPIENT_ENDPOINT, - 0, chan, addr, + 0, rsh_usb_addr.wvalue, rsh_usb_addr.windex, (unsigned char *)&dev->ctrl_data, sizeof(dev->ctrl_data), RSHIM_USB_TIMEOUT); @@ -602,6 +681,9 @@ case USB_BLUEFIELD_2_PRODUCT_ID: bd->ver_id = RSHIM_BLUEFIELD_2; break; + case USB_BLUEFIELD_3_PRODUCT_ID: + bd->ver_id = RSHIM_BLUEFIELD_3; + break; default: bd->ver_id = RSHIM_BLUEFIELD_1; }