Author: arybchik Date: Thu Nov 22 14:31:35 2018 New Revision: 340766 URL: https://svnweb.freebsd.org/changeset/base/340766
Log: sfxge(4): support packed stream Rx mode in libefx Submitted by: Artem V. Andreev <artem.andr...@oktetlabs.ru> Sponsored by: Solarflare Communications, Inc. Differential Revision: https://reviews.freebsd.org/D18022 Modified: head/sys/dev/sfxge/common/ef10_ev.c head/sys/dev/sfxge/common/ef10_impl.h head/sys/dev/sfxge/common/ef10_nic.c head/sys/dev/sfxge/common/ef10_rx.c head/sys/dev/sfxge/common/efsys.h head/sys/dev/sfxge/common/efx.h head/sys/dev/sfxge/common/efx_check.h head/sys/dev/sfxge/common/efx_impl.h head/sys/dev/sfxge/common/efx_nic.c head/sys/dev/sfxge/common/efx_regs_ef10.h head/sys/dev/sfxge/common/efx_rx.c head/sys/dev/sfxge/common/siena_nic.c Modified: head/sys/dev/sfxge/common/ef10_ev.c ============================================================================== --- head/sys/dev/sfxge/common/ef10_ev.c Thu Nov 22 14:10:46 2018 (r340765) +++ head/sys/dev/sfxge/common/ef10_ev.c Thu Nov 22 14:31:35 2018 (r340766) @@ -762,8 +762,90 @@ ef10_ev_qstats_update( } #endif /* EFSYS_OPT_QSTATS */ +#if EFSYS_OPT_RX_PACKED_STREAM static __checkReturn boolean_t +ef10_ev_rx_packed_stream( + __in efx_evq_t *eep, + __in efx_qword_t *eqp, + __in const efx_ev_callbacks_t *eecp, + __in_opt void *arg) +{ + uint32_t label; + uint32_t pkt_count_lbits; + uint16_t flags; + boolean_t should_abort; + efx_evq_rxq_state_t *eersp; + unsigned int pkt_count; + unsigned int current_id; + boolean_t new_buffer; + + pkt_count_lbits = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_DSC_PTR_LBITS); + label = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_QLABEL); + new_buffer = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_EV_ROTATE); + + flags = 0; + + eersp = &eep->ee_rxq_state[label]; + + /* + * RX_DSC_PTR_LBITS has least significant bits of the global + * (not per-buffer) packet counter. It is guaranteed that + * maximum number of completed packets fits in lbits-mask. + * So, modulo lbits-mask arithmetic should be used to calculate + * packet counter increment. + */ + pkt_count = (pkt_count_lbits - eersp->eers_rx_stream_npackets) & + EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS); + eersp->eers_rx_stream_npackets += pkt_count; + + if (new_buffer) { + flags |= EFX_PKT_PACKED_STREAM_NEW_BUFFER; + eersp->eers_rx_packed_stream_credits++; + eersp->eers_rx_read_ptr++; + } + current_id = eersp->eers_rx_read_ptr & eersp->eers_rx_mask; + + /* Check for errors that invalidate checksum and L3/L4 fields */ + if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_ECC_ERR) != 0) { + /* RX frame truncated (error flag is misnamed) */ + EFX_EV_QSTAT_INCR(eep, EV_RX_FRM_TRUNC); + flags |= EFX_DISCARD; + goto deliver; + } + if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_ECRC_ERR) != 0) { + /* Bad Ethernet frame CRC */ + EFX_EV_QSTAT_INCR(eep, EV_RX_ETH_CRC_ERR); + flags |= EFX_DISCARD; + goto deliver; + } + + if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_PARSE_INCOMPLETE)) { + flags |= EFX_PKT_PACKED_STREAM_PARSE_INCOMPLETE; + goto deliver; + } + + if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_IPCKSUM_ERR)) + EFX_EV_QSTAT_INCR(eep, EV_RX_IPV4_HDR_CHKSUM_ERR); + + if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_TCPUDP_CKSUM_ERR)) + EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_UDP_CHKSUM_ERR); + +deliver: + /* If we're not discarding the packet then it is ok */ + if (~flags & EFX_DISCARD) + EFX_EV_QSTAT_INCR(eep, EV_RX_OK); + + EFSYS_ASSERT(eecp->eec_rx_ps != NULL); + should_abort = eecp->eec_rx_ps(arg, label, current_id, pkt_count, + flags); + + return (should_abort); +} + +#endif /* EFSYS_OPT_RX_PACKED_STREAM */ + +static __checkReturn boolean_t ef10_ev_rx( __in efx_evq_t *eep, __in efx_qword_t *eqp, @@ -792,9 +874,20 @@ ef10_ev_rx( return (B_FALSE); /* Basic packet information */ + label = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_QLABEL); + eersp = &eep->ee_rxq_state[label]; + +#if EFSYS_OPT_RX_PACKED_STREAM + /* + * Packed stream events are very different, + * so handle them separately + */ + if (eersp->eers_rx_packed_stream) + return (ef10_ev_rx_packed_stream(eep, eqp, eecp, arg)); +#endif + size = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_BYTES); next_read_lbits = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_DSC_PTR_LBITS); - label = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_QLABEL); eth_tag_class = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_ETH_TAG_CLASS); mac_class = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_MAC_CLASS); l3_class = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_L3_CLASS); @@ -824,7 +917,6 @@ ef10_ev_rx( flags |= EFX_PKT_UNICAST; /* Increment the count of descriptors read */ - eersp = &eep->ee_rxq_state[label]; desc_count = (next_read_lbits - eersp->eers_rx_read_ptr) & EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS); eersp->eers_rx_read_ptr += desc_count; @@ -1246,7 +1338,8 @@ ef10_ev_mcdi( ef10_ev_rxlabel_init( __in efx_evq_t *eep, __in efx_rxq_t *erp, - __in unsigned int label) + __in unsigned int label, + __in boolean_t packed_stream) { efx_evq_rxq_state_t *eersp; @@ -1255,8 +1348,41 @@ ef10_ev_rxlabel_init( EFSYS_ASSERT3U(eersp->eers_rx_mask, ==, 0); +#if EFSYS_OPT_RX_PACKED_STREAM + /* + * For packed stream modes, the very first event will + * have a new buffer flag set, so it will be incremented, + * yielding the correct pointer. That results in a simpler + * code than trying to detect start-of-the-world condition + * in the event handler. + */ + eersp->eers_rx_read_ptr = packed_stream ? ~0 : 0; +#else eersp->eers_rx_read_ptr = 0; +#endif eersp->eers_rx_mask = erp->er_mask; +#if EFSYS_OPT_RX_PACKED_STREAM + eersp->eers_rx_stream_npackets = 0; + eersp->eers_rx_packed_stream = packed_stream; + if (packed_stream) { + eersp->eers_rx_packed_stream_credits = (eep->ee_mask + 1) / + EFX_DIV_ROUND_UP(EFX_RX_PACKED_STREAM_MEM_PER_CREDIT, + EFX_RX_PACKED_STREAM_MIN_PACKET_SPACE); + EFSYS_ASSERT3U(eersp->eers_rx_packed_stream_credits, !=, 0); + /* + * A single credit is allocated to the queue when it is started. + * It is immediately spent by the first packet which has NEW + * BUFFER flag set, though, but still we shall take into + * account, as to not wrap around the maximum number of credits + * accidentally + */ + eersp->eers_rx_packed_stream_credits--; + EFSYS_ASSERT3U(eersp->eers_rx_packed_stream_credits, <=, + EFX_RX_PACKED_STREAM_MAX_CREDITS); + } +#else + EFSYS_ASSERT(!packed_stream); +#endif } void @@ -1273,6 +1399,11 @@ ef10_ev_rxlabel_fini( eersp->eers_rx_read_ptr = 0; eersp->eers_rx_mask = 0; +#if EFSYS_OPT_RX_PACKED_STREAM + eersp->eers_rx_stream_npackets = 0; + eersp->eers_rx_packed_stream = B_FALSE; + eersp->eers_rx_packed_stream_credits = 0; +#endif } #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ Modified: head/sys/dev/sfxge/common/ef10_impl.h ============================================================================== --- head/sys/dev/sfxge/common/ef10_impl.h Thu Nov 22 14:10:46 2018 (r340765) +++ head/sys/dev/sfxge/common/ef10_impl.h Thu Nov 22 14:31:35 2018 (r340766) @@ -118,7 +118,8 @@ ef10_ev_qstats_update( ef10_ev_rxlabel_init( __in efx_evq_t *eep, __in efx_rxq_t *erp, - __in unsigned int label); + __in unsigned int label, + __in boolean_t packed_stream); void ef10_ev_rxlabel_fini( @@ -685,6 +686,22 @@ ef10_tx_qpush( __in unsigned int added, __in unsigned int pushed); +#if EFSYS_OPT_RX_PACKED_STREAM +extern void +ef10_rx_qpush_ps_credits( + __in efx_rxq_t *erp); + +extern __checkReturn uint8_t * +ef10_rx_qps_packet_info( + __in efx_rxq_t *erp, + __in uint8_t *buffer, + __in uint32_t buffer_length, + __in uint32_t current_offset, + __out uint16_t *lengthp, + __out uint32_t *next_offsetp, + __out uint32_t *timestamp); +#endif + extern __checkReturn efx_rc_t ef10_tx_qpace( __in efx_txq_t *etp, @@ -1131,6 +1148,35 @@ ef10_external_port_mapping( __in uint32_t port, __out uint8_t *external_portp); +#if EFSYS_OPT_RX_PACKED_STREAM + +/* Data space per credit in packed stream mode */ +#define EFX_RX_PACKED_STREAM_MEM_PER_CREDIT (1 << 16) + +/* + * Received packets are always aligned at this boundary. Also there always + * exists a gap of this size between packets. + * (see SF-112241-TC, 4.5) + */ +#define EFX_RX_PACKED_STREAM_ALIGNMENT 64 + +/* + * Size of a pseudo-header prepended to received packets + * in packed stream mode + */ +#define EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE 8 + +/* Minimum space for packet in packed stream mode */ +#define EFX_RX_PACKED_STREAM_MIN_PACKET_SPACE \ + P2ROUNDUP(EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE + \ + EFX_MAC_PDU_MIN + \ + EFX_RX_PACKED_STREAM_ALIGNMENT, \ + EFX_RX_PACKED_STREAM_ALIGNMENT) + +/* Maximum number of credits */ +#define EFX_RX_PACKED_STREAM_MAX_CREDITS 127 + +#endif /* EFSYS_OPT_RX_PACKED_STREAM */ #ifdef __cplusplus } Modified: head/sys/dev/sfxge/common/ef10_nic.c ============================================================================== --- head/sys/dev/sfxge/common/ef10_nic.c Thu Nov 22 14:10:46 2018 (r340765) +++ head/sys/dev/sfxge/common/ef10_nic.c Thu Nov 22 14:31:35 2018 (r340766) @@ -1072,6 +1072,17 @@ ef10_get_datapath_caps( encp->enc_rx_disable_scatter_supported = CAP_FLAG(flags, RX_DISABLE_SCATTER) ? B_TRUE : B_FALSE; + /* Check if the firmware supports packed stream mode */ + encp->enc_rx_packed_stream_supported = + CAP_FLAG(flags, RX_PACKED_STREAM) ? B_TRUE : B_FALSE; + + /* + * Check if the firmware supports configurable buffer sizes + * for packed stream mode (otherwise buffer size is 1Mbyte) + */ + encp->enc_rx_var_packed_stream_supported = + CAP_FLAG(flags, RX_PACKED_STREAM_VAR_BUFFERS) ? B_TRUE : B_FALSE; + /* Check if the firmware supports set mac with running filters */ encp->enc_allow_set_mac_with_installed_filters = CAP_FLAG(flags, VADAPTOR_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED) ? Modified: head/sys/dev/sfxge/common/ef10_rx.c ============================================================================== --- head/sys/dev/sfxge/common/ef10_rx.c Thu Nov 22 14:10:46 2018 (r340765) +++ head/sys/dev/sfxge/common/ef10_rx.c Thu Nov 22 14:31:35 2018 (r340766) @@ -46,41 +46,51 @@ efx_mcdi_init_rxq( __in uint32_t label, __in uint32_t instance, __in efsys_mem_t *esmp, - __in boolean_t disable_scatter) + __in boolean_t disable_scatter, + __in uint32_t ps_bufsize) { efx_mcdi_req_t req; - uint8_t payload[ - MC_CMD_INIT_RXQ_IN_LEN(EFX_RXQ_NBUFS(EFX_RXQ_MAXNDESCS))]; + uint8_t payload[MAX(MC_CMD_INIT_RXQ_EXT_IN_LEN, + MC_CMD_INIT_RXQ_EXT_OUT_LEN)]; int npages = EFX_RXQ_NBUFS(size); int i; efx_qword_t *dma_addr; uint64_t addr; efx_rc_t rc; + uint32_t dma_mode; /* If this changes, then the payload size might need to change. */ EFSYS_ASSERT3U(MC_CMD_INIT_RXQ_OUT_LEN, ==, 0); EFSYS_ASSERT3U(size, <=, EFX_RXQ_MAXNDESCS); + if (ps_bufsize > 0) + dma_mode = MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM; + else + dma_mode = MC_CMD_INIT_RXQ_EXT_IN_SINGLE_PACKET; + (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_INIT_RXQ; req.emr_in_buf = payload; - req.emr_in_length = MC_CMD_INIT_RXQ_IN_LEN(npages); + req.emr_in_length = MC_CMD_INIT_RXQ_EXT_IN_LEN; req.emr_out_buf = payload; - req.emr_out_length = MC_CMD_INIT_RXQ_OUT_LEN; + req.emr_out_length = MC_CMD_INIT_RXQ_EXT_OUT_LEN; - MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_SIZE, size); - MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_TARGET_EVQ, target_evq); - MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_LABEL, label); - MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_INSTANCE, instance); - MCDI_IN_POPULATE_DWORD_6(req, INIT_RXQ_IN_FLAGS, - INIT_RXQ_IN_FLAG_BUFF_MODE, 0, - INIT_RXQ_IN_FLAG_HDR_SPLIT, 0, - INIT_RXQ_IN_FLAG_TIMESTAMP, 0, - INIT_RXQ_IN_CRC_MODE, 0, - INIT_RXQ_IN_FLAG_PREFIX, 1, - INIT_RXQ_IN_FLAG_DISABLE_SCATTER, disable_scatter); - MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_OWNER_ID, 0); - MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_PORT_ID, EVB_PORT_ID_ASSIGNED); + MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_SIZE, size); + MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, target_evq); + MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_LABEL, label); + MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_INSTANCE, instance); + MCDI_IN_POPULATE_DWORD_8(req, INIT_RXQ_EXT_IN_FLAGS, + INIT_RXQ_EXT_IN_FLAG_BUFF_MODE, 0, + INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT, 0, + INIT_RXQ_EXT_IN_FLAG_TIMESTAMP, 0, + INIT_RXQ_EXT_IN_CRC_MODE, 0, + INIT_RXQ_EXT_IN_FLAG_PREFIX, 1, + INIT_RXQ_EXT_IN_FLAG_DISABLE_SCATTER, disable_scatter, + INIT_RXQ_EXT_IN_DMA_MODE, + dma_mode, + INIT_RXQ_EXT_IN_PACKED_STREAM_BUFF_SIZE, ps_bufsize); + MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_OWNER_ID, 0); + MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_PORT_ID, EVB_PORT_ID_ASSIGNED); dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR); addr = EFSYS_MEM_ADDR(esmp); @@ -707,6 +717,95 @@ ef10_rx_qpush( erp->er_index, &dword, B_FALSE); } +#if EFSYS_OPT_RX_PACKED_STREAM + + void +ef10_rx_qpush_ps_credits( + __in efx_rxq_t *erp) +{ + efx_nic_t *enp = erp->er_enp; + efx_dword_t dword; + efx_evq_rxq_state_t *rxq_state = + &erp->er_eep->ee_rxq_state[erp->er_label]; + uint32_t credits; + + EFSYS_ASSERT(rxq_state->eers_rx_packed_stream); + + if (rxq_state->eers_rx_packed_stream_credits == 0) + return; + + /* + * It is a bug if we think that FW has utilized more + * credits than it is allowed to have (maximum). However, + * make sure that we do not credit more than maximum anyway. + */ + credits = MIN(rxq_state->eers_rx_packed_stream_credits, + EFX_RX_PACKED_STREAM_MAX_CREDITS); + EFX_POPULATE_DWORD_3(dword, + ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1, + ERF_DZ_RX_DESC_MAGIC_CMD, + ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS, + ERF_DZ_RX_DESC_MAGIC_DATA, credits); + EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG, + erp->er_index, &dword, B_FALSE); + + rxq_state->eers_rx_packed_stream_credits = 0; +} + +/* + * In accordance with SF-112241-TC the received data has the following layout: + * - 8 byte pseudo-header which consist of: + * - 4 byte little-endian timestamp + * - 2 byte little-endian captured length in bytes + * - 2 byte little-endian original packet length in bytes + * - captured packet bytes + * - optional padding to align to 64 bytes boundary + * - 64 bytes scratch space for the host software + */ + __checkReturn uint8_t * +ef10_rx_qps_packet_info( + __in efx_rxq_t *erp, + __in uint8_t *buffer, + __in uint32_t buffer_length, + __in uint32_t current_offset, + __out uint16_t *lengthp, + __out uint32_t *next_offsetp, + __out uint32_t *timestamp) +{ + uint16_t buf_len; + uint8_t *pkt_start; + efx_qword_t *qwordp; + efx_evq_rxq_state_t *rxq_state = + &erp->er_eep->ee_rxq_state[erp->er_label]; + + EFSYS_ASSERT(rxq_state->eers_rx_packed_stream); + + buffer += current_offset; + pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE; + + qwordp = (efx_qword_t *)buffer; + *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP); + *lengthp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN); + buf_len = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN); + + buf_len = P2ROUNDUP(buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE, + EFX_RX_PACKED_STREAM_ALIGNMENT); + *next_offsetp = + current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT; + + EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length); + EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp); + + if ((*next_offsetp ^ current_offset) & + EFX_RX_PACKED_STREAM_MEM_PER_CREDIT) + rxq_state->eers_rx_packed_stream_credits++; + + return (pkt_start); +} + + +#endif + __checkReturn efx_rc_t ef10_rx_qflush( __in efx_rxq_t *erp) @@ -749,6 +848,7 @@ ef10_rx_qcreate( efx_nic_cfg_t *encp = &(enp->en_nic_cfg); efx_rc_t rc; boolean_t disable_scatter; + unsigned int ps_buf_size; _NOTE(ARGUNUSED(id, erp)) @@ -768,6 +868,51 @@ ef10_rx_qcreate( goto fail2; } + switch (type) { + case EFX_RXQ_TYPE_DEFAULT: + case EFX_RXQ_TYPE_SCATTER: + ps_buf_size = 0; + break; +#if EFSYS_OPT_RX_PACKED_STREAM + case EFX_RXQ_TYPE_PACKED_STREAM_1M: + ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M; + break; + case EFX_RXQ_TYPE_PACKED_STREAM_512K: + ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K; + break; + case EFX_RXQ_TYPE_PACKED_STREAM_256K: + ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K; + break; + case EFX_RXQ_TYPE_PACKED_STREAM_128K: + ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K; + break; + case EFX_RXQ_TYPE_PACKED_STREAM_64K: + ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K; + break; +#endif /* EFSYS_OPT_RX_PACKED_STREAM */ + default: + rc = ENOTSUP; + goto fail3; + } + +#if EFSYS_OPT_RX_PACKED_STREAM + if (ps_buf_size != 0) { + /* Check if datapath firmware supports packed stream mode */ + if (encp->enc_rx_packed_stream_supported == B_FALSE) { + rc = ENOTSUP; + goto fail4; + } + /* Check if packed stream allows configurable buffer sizes */ + if ((type != EFX_RXQ_TYPE_PACKED_STREAM_1M) && + (encp->enc_rx_var_packed_stream_supported == B_FALSE)) { + rc = ENOTSUP; + goto fail5; + } + } +#else /* EFSYS_OPT_RX_PACKED_STREAM */ + EFSYS_ASSERT(ps_buf_size == 0); +#endif /* EFSYS_OPT_RX_PACKED_STREAM */ + /* Scatter can only be disabled if the firmware supports doing so */ if (type == EFX_RXQ_TYPE_SCATTER) disable_scatter = B_FALSE; @@ -775,16 +920,24 @@ ef10_rx_qcreate( disable_scatter = encp->enc_rx_disable_scatter_supported; if ((rc = efx_mcdi_init_rxq(enp, n, eep->ee_index, label, index, - esmp, disable_scatter)) != 0) - goto fail3; + esmp, disable_scatter, ps_buf_size)) != 0) + goto fail6; erp->er_eep = eep; erp->er_label = label; - ef10_ev_rxlabel_init(eep, erp, label); + ef10_ev_rxlabel_init(eep, erp, label, ps_buf_size != 0); return (0); +fail6: + EFSYS_PROBE(fail6); +#if EFSYS_OPT_RX_PACKED_STREAM +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); +#endif /* EFSYS_OPT_RX_PACKED_STREAM */ fail3: EFSYS_PROBE(fail3); fail2: Modified: head/sys/dev/sfxge/common/efsys.h ============================================================================== --- head/sys/dev/sfxge/common/efsys.h Thu Nov 22 14:10:46 2018 (r340765) +++ head/sys/dev/sfxge/common/efsys.h Thu Nov 22 14:31:35 2018 (r340766) @@ -281,6 +281,8 @@ sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t ma #define EFSYS_OPT_ALLOW_UNCONFIGURED_NIC 0 +#define EFSYS_OPT_RX_PACKED_STREAM 0 + /* ID */ typedef struct __efsys_identifier_s efsys_identifier_t; Modified: head/sys/dev/sfxge/common/efx.h ============================================================================== --- head/sys/dev/sfxge/common/efx.h Thu Nov 22 14:10:46 2018 (r340765) +++ head/sys/dev/sfxge/common/efx.h Thu Nov 22 14:31:35 2018 (r340766) @@ -52,6 +52,9 @@ extern "C" { #define EFX_FIELD_OFFSET(_type, _field) \ ((size_t) &(((_type *)0)->_field)) +/* The macro expands divider twice */ +#define EFX_DIV_ROUND_UP(_n, _d) (((_n) + (_d) - 1) / (_d)) + /* Return codes */ typedef __success(return == 0) int efx_rc_t; @@ -1086,6 +1089,7 @@ efx_bist_stop( #define EFX_FEATURE_PIO_BUFFERS 0x00000800 #define EFX_FEATURE_FW_ASSISTED_TSO 0x00001000 #define EFX_FEATURE_FW_ASSISTED_TSO_V2 0x00002000 +#define EFX_FEATURE_PACKED_STREAM 0x00004000 typedef struct efx_nic_cfg_s { uint32_t enc_board_type; @@ -1181,6 +1185,8 @@ typedef struct efx_nic_cfg_s { boolean_t enc_allow_set_mac_with_installed_filters; boolean_t enc_enhanced_set_mac_supported; boolean_t enc_init_evq_v2_supported; + boolean_t enc_rx_packed_stream_supported; + boolean_t enc_rx_var_packed_stream_supported; boolean_t enc_pm_and_rxdp_counters; boolean_t enc_mac_stats_40g_tx_size_bins; /* External port identifier */ @@ -1622,6 +1628,16 @@ typedef __checkReturn boolean_t #define EFX_ADDR_MISMATCH 0x4000 #define EFX_DISCARD 0x8000 +/* + * The following flags are used only for packed stream + * mode. The values for the flags are reused to fit into 16 bit, + * since EFX_PKT_START and EFX_PKT_CONT are never used in + * packed stream mode + */ +#define EFX_PKT_PACKED_STREAM_NEW_BUFFER EFX_PKT_START +#define EFX_PKT_PACKED_STREAM_PARSE_INCOMPLETE EFX_PKT_CONT + + #define EFX_EV_RX_NLABELS 32 #define EFX_EV_TX_NLABELS 32 @@ -1633,7 +1649,29 @@ typedef __checkReturn boolean_t __in uint32_t size, __in uint16_t flags); +#if EFSYS_OPT_RX_PACKED_STREAM + +/* + * Packed stream mode is documented in SF-112241-TC. + * The general idea is that, instead of putting each incoming + * packet into a separate buffer which is specified in a RX + * descriptor, a large buffer is provided to the hardware and + * packets are put there in a continuous stream. + * The main advantage of such an approach is that RX queue refilling + * happens much less frequently. + */ + typedef __checkReturn boolean_t +(*efx_rx_ps_ev_t)( + __in_opt void *arg, + __in uint32_t label, + __in uint32_t id, + __in uint32_t pkt_count, + __in uint16_t flags); + +#endif + +typedef __checkReturn boolean_t (*efx_tx_ev_t)( __in_opt void *arg, __in uint32_t label, @@ -1722,6 +1760,9 @@ typedef __checkReturn boolean_t typedef struct efx_ev_callbacks_s { efx_initialized_ev_t eec_initialized; efx_rx_ev_t eec_rx; +#if EFSYS_OPT_RX_PACKED_STREAM + efx_rx_ps_ev_t eec_rx_ps; +#endif efx_tx_ev_t eec_tx; efx_exception_ev_t eec_exception; efx_rxq_flush_done_ev_t eec_rxq_flush_done; @@ -1900,6 +1941,11 @@ efx_pseudo_hdr_pkt_length_get( typedef enum efx_rxq_type_e { EFX_RXQ_TYPE_DEFAULT, EFX_RXQ_TYPE_SCATTER, + EFX_RXQ_TYPE_PACKED_STREAM_1M, + EFX_RXQ_TYPE_PACKED_STREAM_512K, + EFX_RXQ_TYPE_PACKED_STREAM_256K, + EFX_RXQ_TYPE_PACKED_STREAM_128K, + EFX_RXQ_TYPE_PACKED_STREAM_64K, EFX_RXQ_NTYPES } efx_rxq_type_t; @@ -1939,6 +1985,29 @@ efx_rx_qpush( __in efx_rxq_t *erp, __in unsigned int added, __inout unsigned int *pushedp); + +#if EFSYS_OPT_RX_PACKED_STREAM + +/* + * Fake length for RXQ descriptors in packed stream mode + * to make hardware happy + */ +#define EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE 32 + +extern void +efx_rx_qpush_ps_credits( + __in efx_rxq_t *erp); + +extern __checkReturn uint8_t * +efx_rx_qps_packet_info( + __in efx_rxq_t *erp, + __in uint8_t *buffer, + __in uint32_t buffer_length, + __in uint32_t current_offset, + __out uint16_t *lengthp, + __out uint32_t *next_offsetp, + __out uint32_t *timestamp); +#endif extern __checkReturn efx_rc_t efx_rx_qflush( Modified: head/sys/dev/sfxge/common/efx_check.h ============================================================================== --- head/sys/dev/sfxge/common/efx_check.h Thu Nov 22 14:10:46 2018 (r340765) +++ head/sys/dev/sfxge/common/efx_check.h Thu Nov 22 14:31:35 2018 (r340766) @@ -339,5 +339,11 @@ # endif #endif /* EFSYS_OPT_ALLOW_UNCONFIGURED_NIC */ +/* Support packed stream mode */ +#if EFSYS_OPT_RX_PACKED_STREAM +# if !(EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) +# error "PACKED_STREAM requires HUNTINGTON or MEDFORD" +# endif +#endif #endif /* _SYS_EFX_CHECK_H */ Modified: head/sys/dev/sfxge/common/efx_impl.h ============================================================================== --- head/sys/dev/sfxge/common/efx_impl.h Thu Nov 22 14:10:46 2018 (r340765) +++ head/sys/dev/sfxge/common/efx_impl.h Thu Nov 22 14:31:35 2018 (r340766) @@ -170,6 +170,12 @@ typedef struct efx_rx_ops_s { unsigned int, unsigned int, unsigned int); void (*erxo_qpush)(efx_rxq_t *, unsigned int, unsigned int *); +#if EFSYS_OPT_RX_PACKED_STREAM + void (*erxo_qpush_ps_credits)(efx_rxq_t *); + uint8_t * (*erxo_qps_packet_info)(efx_rxq_t *, uint8_t *, + uint32_t, uint32_t, + uint16_t *, uint32_t *, uint32_t *); +#endif efx_rc_t (*erxo_qflush)(efx_rxq_t *); void (*erxo_qenable)(efx_rxq_t *); efx_rc_t (*erxo_qcreate)(efx_nic_t *enp, unsigned int, @@ -694,6 +700,11 @@ typedef boolean_t (*efx_ev_handler_t)(efx_evq_t *, efx typedef struct efx_evq_rxq_state_s { unsigned int eers_rx_read_ptr; unsigned int eers_rx_mask; +#if EFSYS_OPT_RX_PACKED_STREAM + unsigned int eers_rx_stream_npackets; + boolean_t eers_rx_packed_stream; + unsigned int eers_rx_packed_stream_credits; +#endif } efx_evq_rxq_state_t; struct efx_evq_s { Modified: head/sys/dev/sfxge/common/efx_nic.c ============================================================================== --- head/sys/dev/sfxge/common/efx_nic.c Thu Nov 22 14:10:46 2018 (r340765) +++ head/sys/dev/sfxge/common/efx_nic.c Thu Nov 22 14:31:35 2018 (r340766) @@ -289,7 +289,8 @@ efx_nic_create( EFX_FEATURE_MCDI_DMA | EFX_FEATURE_PIO_BUFFERS | EFX_FEATURE_FW_ASSISTED_TSO | - EFX_FEATURE_FW_ASSISTED_TSO_V2; + EFX_FEATURE_FW_ASSISTED_TSO_V2 | + EFX_FEATURE_PACKED_STREAM; break; #endif /* EFSYS_OPT_HUNTINGTON */ @@ -308,7 +309,8 @@ efx_nic_create( EFX_FEATURE_MAC_HEADER_FILTERS | EFX_FEATURE_MCDI_DMA | EFX_FEATURE_PIO_BUFFERS | - EFX_FEATURE_FW_ASSISTED_TSO_V2; + EFX_FEATURE_FW_ASSISTED_TSO_V2 | + EFX_FEATURE_PACKED_STREAM; break; #endif /* EFSYS_OPT_MEDFORD */ Modified: head/sys/dev/sfxge/common/efx_regs_ef10.h ============================================================================== --- head/sys/dev/sfxge/common/efx_regs_ef10.h Thu Nov 22 14:10:46 2018 (r340765) +++ head/sys/dev/sfxge/common/efx_regs_ef10.h Thu Nov 22 14:31:35 2018 (r340766) @@ -169,7 +169,6 @@ extern "C" { #define ERF_DZ_RX_DESC_WPTR_LBN 0 #define ERF_DZ_RX_DESC_WPTR_WIDTH 12 - /* * TX_DESC_UPD_REG(96bit): * @@ -543,6 +542,31 @@ extern "C" { #define ERF_DD_EVQ_IND_TIMER_VAL_LBN 0 #define ERF_DD_EVQ_IND_TIMER_VAL_WIDTH 8 +/* Packed stream magic doorbell command */ +#define ERF_DZ_RX_DESC_MAGIC_DOORBELL_LBN 11 +#define ERF_DZ_RX_DESC_MAGIC_DOORBELL_WIDTH 1 + +#define ERF_DZ_RX_DESC_MAGIC_CMD_LBN 8 +#define ERF_DZ_RX_DESC_MAGIC_CMD_WIDTH 3 +#define ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS 0 + +#define ERF_DZ_RX_DESC_MAGIC_DATA_LBN 0 +#define ERF_DZ_RX_DESC_MAGIC_DATA_WIDTH 8 + +/* Packed stream RX packet prefix */ +#define ES_DZ_PS_RX_PREFIX_TSTAMP_LBN 0 +#define ES_DZ_PS_RX_PREFIX_TSTAMP_WIDTH 32 +#define ES_DZ_PS_RX_PREFIX_CAP_LEN_LBN 32 +#define ES_DZ_PS_RX_PREFIX_CAP_LEN_WIDTH 16 +#define ES_DZ_PS_RX_PREFIX_ORIG_LEN_LBN 48 +#define ES_DZ_PS_RX_PREFIX_ORIG_LEN_WIDTH 16 + +/* + * An extra flag for the packed stream mode, + * signalling the start of a new buffer + */ +#define ESF_DZ_RX_EV_ROTATE_LBN 53 +#define ESF_DZ_RX_EV_ROTATE_WIDTH 1 #ifdef __cplusplus } Modified: head/sys/dev/sfxge/common/efx_rx.c ============================================================================== --- head/sys/dev/sfxge/common/efx_rx.c Thu Nov 22 14:10:46 2018 (r340765) +++ head/sys/dev/sfxge/common/efx_rx.c Thu Nov 22 14:31:35 2018 (r340766) @@ -103,6 +103,22 @@ siena_rx_qpush( __in unsigned int added, __inout unsigned int *pushedp); +#if EFSYS_OPT_RX_PACKED_STREAM +static void +siena_rx_qpush_ps_credits( + __in efx_rxq_t *erp); + +static __checkReturn uint8_t * +siena_rx_qps_packet_info( + __in efx_rxq_t *erp, + __in uint8_t *buffer, + __in uint32_t buffer_length, + __in uint32_t current_offset, + __out uint16_t *lengthp, + __out uint32_t *next_offsetp, + __out uint32_t *timestamp); +#endif + static __checkReturn efx_rc_t siena_rx_qflush( __in efx_rxq_t *erp); @@ -146,6 +162,10 @@ static const efx_rx_ops_t __efx_rx_siena_ops = { siena_rx_prefix_pktlen, /* erxo_prefix_pktlen */ siena_rx_qpost, /* erxo_qpost */ siena_rx_qpush, /* erxo_qpush */ +#if EFSYS_OPT_RX_PACKED_STREAM + siena_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */ + siena_rx_qps_packet_info, /* erxo_qps_packet_info */ +#endif siena_rx_qflush, /* erxo_qflush */ siena_rx_qenable, /* erxo_qenable */ siena_rx_qcreate, /* erxo_qcreate */ @@ -169,6 +189,10 @@ static const efx_rx_ops_t __efx_rx_ef10_ops = { ef10_rx_prefix_pktlen, /* erxo_prefix_pktlen */ ef10_rx_qpost, /* erxo_qpost */ ef10_rx_qpush, /* erxo_qpush */ +#if EFSYS_OPT_RX_PACKED_STREAM + ef10_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */ + ef10_rx_qps_packet_info, /* erxo_qps_packet_info */ +#endif ef10_rx_qflush, /* erxo_qflush */ ef10_rx_qenable, /* erxo_qenable */ ef10_rx_qcreate, /* erxo_qcreate */ @@ -430,7 +454,41 @@ efx_rx_qpost( erxop->erxo_qpost(erp, addrp, size, n, completed, added); } +#if EFSYS_OPT_RX_PACKED_STREAM + void +efx_rx_qpush_ps_credits( + __in efx_rxq_t *erp) +{ + efx_nic_t *enp = erp->er_enp; + const efx_rx_ops_t *erxop = enp->en_erxop; + + EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); + + erxop->erxo_qpush_ps_credits(erp); +} + + __checkReturn uint8_t * +efx_rx_qps_packet_info( + __in efx_rxq_t *erp, + __in uint8_t *buffer, + __in uint32_t buffer_length, + __in uint32_t current_offset, + __out uint16_t *lengthp, + __out uint32_t *next_offsetp, + __out uint32_t *timestamp) +{ + efx_nic_t *enp = erp->er_enp; + const efx_rx_ops_t *erxop = enp->en_erxop; + + return (erxop->erxo_qps_packet_info(erp, buffer, + buffer_length, current_offset, lengthp, + next_offsetp, timestamp)); +} + +#endif /* EFSYS_OPT_RX_PACKED_STREAM */ + + void efx_rx_qpush( __in efx_rxq_t *erp, __in unsigned int added, @@ -1075,6 +1133,32 @@ siena_rx_qpush( EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0, erp->er_index, &dword, B_FALSE); } + +#if EFSYS_OPT_RX_PACKED_STREAM +static void +siena_rx_qpush_ps_credits( + __in efx_rxq_t *erp) +{ + /* Not supported by Siena hardware */ + EFSYS_ASSERT(0); +} + +static uint8_t * +siena_rx_qps_packet_info( + __in efx_rxq_t *erp, + __in uint8_t *buffer, + __in uint32_t buffer_length, + __in uint32_t current_offset, + __out uint16_t *lengthp, + __out uint32_t *next_offsetp, + __out uint32_t *timestamp) +{ + /* Not supported by Siena hardware */ + EFSYS_ASSERT(0); + + return (NULL); +} +#endif /* EFSYS_OPT_RX_PACKED_STREAM */ static __checkReturn efx_rc_t siena_rx_qflush( Modified: head/sys/dev/sfxge/common/siena_nic.c ============================================================================== --- head/sys/dev/sfxge/common/siena_nic.c Thu Nov 22 14:10:46 2018 (r340765) +++ head/sys/dev/sfxge/common/siena_nic.c Thu Nov 22 14:31:35 2018 (r340766) @@ -167,6 +167,8 @@ siena_board_cfg( encp->enc_fw_assisted_tso_v2_enabled = B_FALSE; encp->enc_fw_assisted_tso_v2_n_contexts = 0; encp->enc_allow_set_mac_with_installed_filters = B_TRUE; + encp->enc_rx_packed_stream_supported = B_FALSE; + encp->enc_rx_var_packed_stream_supported = B_FALSE; /* Siena supports two 10G ports, and 8 lanes of PCIe Gen2 */ encp->enc_required_pcie_bandwidth_mbps = 2 * 10000; _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
