Module Name: src
Committed By: ryo
Date: Wed Jan 1 10:11:21 UTC 2020
Modified Files:
src/distrib/sets/lists/man: mi
src/share/man/man4: Makefile
src/sys/dev/pci: files.pci
Added Files:
src/share/man/man4: aq.4
src/sys/dev/pci: if_aq.c
Log Message:
add support Aquantia AQC seriese 10G network adapters.
this driver is based on the FreeBSD version
https://github.com/Aquantia/aqtion-freebsd ,
but drastically rewritten for NetBSD.
To generate a diff of this commit:
cvs rdiff -u -r1.1667 -r1.1668 src/distrib/sets/lists/man/mi
cvs rdiff -u -r1.692 -r1.693 src/share/man/man4/Makefile
cvs rdiff -u -r0 -r1.1 src/share/man/man4/aq.4
cvs rdiff -u -r1.418 -r1.419 src/sys/dev/pci/files.pci
cvs rdiff -u -r0 -r1.1 src/sys/dev/pci/if_aq.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/distrib/sets/lists/man/mi
diff -u src/distrib/sets/lists/man/mi:1.1667 src/distrib/sets/lists/man/mi:1.1668
--- src/distrib/sets/lists/man/mi:1.1667 Mon Dec 23 06:45:37 2019
+++ src/distrib/sets/lists/man/mi Wed Jan 1 10:11:21 2020
@@ -1,4 +1,4 @@
-# $NetBSD: mi,v 1.1667 2019/12/23 06:45:37 maxv Exp $
+# $NetBSD: mi,v 1.1668 2020/01/01 10:11:21 ryo Exp $
#
# Note: don't delete entries from here - mark them as "obsolete" instead.
#
@@ -855,6 +855,7 @@
./usr/share/man/cat4/ams.0 man-sys-catman .cat
./usr/share/man/cat4/an.0 man-sys-catman .cat
./usr/share/man/cat4/aps.0 man-sys-catman .cat
+./usr/share/man/cat4/aq.0 man-sys-catman .cat
./usr/share/man/cat4/arc/intro.0 man-sys-catman .cat
./usr/share/man/cat4/arcmsr.0 man-sys-catman .cat
./usr/share/man/cat4/arcofi.0 man-sys-catman .cat
@@ -4077,6 +4078,7 @@
./usr/share/man/html4/ams.html man-sys-htmlman html
./usr/share/man/html4/an.html man-sys-htmlman html
./usr/share/man/html4/aps.html man-sys-htmlman html
+./usr/share/man/html4/aq.html man-sys-htmlman html
./usr/share/man/html4/arc/intro.html man-sys-htmlman html
./usr/share/man/html4/arcmsr.html man-sys-htmlman html
./usr/share/man/html4/arcofi.html man-sys-htmlman html
@@ -6995,6 +6997,7 @@
./usr/share/man/man4/ams.4 man-sys-man .man
./usr/share/man/man4/an.4 man-sys-man .man
./usr/share/man/man4/aps.4 man-sys-man .man
+./usr/share/man/man4/aq.4 man-sys-man .man
./usr/share/man/man4/arc/intro.4 man-sys-man .man
./usr/share/man/man4/arcmsr.4 man-sys-man .man
./usr/share/man/man4/arcofi.4 man-sys-man .man
Index: src/share/man/man4/Makefile
diff -u src/share/man/man4/Makefile:1.692 src/share/man/man4/Makefile:1.693
--- src/share/man/man4/Makefile:1.692 Mon Dec 23 06:45:37 2019
+++ src/share/man/man4/Makefile Wed Jan 1 10:11:21 2020
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile,v 1.692 2019/12/23 06:45:37 maxv Exp $
+# $NetBSD: Makefile,v 1.693 2020/01/01 10:11:21 ryo Exp $
# @(#)Makefile 8.1 (Berkeley) 6/18/93
MAN= aac.4 ac97.4 acardide.4 aceride.4 acphy.4 \
@@ -6,7 +6,7 @@ MAN= aac.4 ac97.4 acardide.4 aceride.4 a
adc.4 adm1026hm.4 admtemp.4 adv.4 adw.4 age.4 agp.4 agr.4 ahb.4 ahc.4 \
ahcisata.4 ahd.4 aibs.4 alc.4 ale.4 alipm.4 altmem.4 altq.4 \
am2315temp.4 amdpm.4 amdtemp.4 amhphy.4 amr.4 aps.4 asus.4 \
- an.4 arcmsr.4 arcofi.4 aria.4 artsata.4 ata.4 atalk.4 ataraid.4 \
+ an.4 aq.4 arcmsr.4 arcofi.4 aria.4 artsata.4 ata.4 atalk.4 ataraid.4 \
ath.4 athn.4 atphy.4 atppc.4 attimer.4 atw.4 \
auacer.4 audio.4 audiocs.4 auich.4 \
auixp.4 autri.4 auvia.4 awi.4 azalia.4 \
Index: src/sys/dev/pci/files.pci
diff -u src/sys/dev/pci/files.pci:1.418 src/sys/dev/pci/files.pci:1.419
--- src/sys/dev/pci/files.pci:1.418 Tue Dec 10 12:08:52 2019
+++ src/sys/dev/pci/files.pci Wed Jan 1 10:11:21 2020
@@ -1,4 +1,4 @@
-# $NetBSD: files.pci,v 1.418 2019/12/10 12:08:52 yamaguchi Exp $
+# $NetBSD: files.pci,v 1.419 2020/01/01 10:11:21 ryo Exp $
#
# Config file and device description for machine-independent PCI code.
# Included by ports that need it. Requires that the SCSI files be
@@ -167,6 +167,12 @@ device mpii: scsi
attach mpii at pci
file dev/pci/mpii.c mpii
+# Aquantia/Atlantic 10-Gigabit Ethernet
+device aq: ether, ifnet, arp
+attach aq at pci
+file dev/pci/if_aq.c aq
+defflag opt_if_aq.h AQ_EVENT_COUNTERS
+
# Ethernet driver for DC21040-based boards
device de: ether, ifnet, arp
attach de at pci
Added files:
Index: src/share/man/man4/aq.4
diff -u /dev/null src/share/man/man4/aq.4:1.1
--- /dev/null Wed Jan 1 10:11:22 2020
+++ src/share/man/man4/aq.4 Wed Jan 1 10:11:21 2020
@@ -0,0 +1,84 @@
+.\" $NetBSD: aq.4,v 1.1 2020/01/01 10:11:21 ryo Exp $
+.\"
+.\" Copyright (c) 2020 Ryo Shimizu <r...@nerv.org>
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\" notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\" notice, this list of conditions and the following disclaimer in the
+.\" documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+.\" WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+.\" DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+.\" INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+.\" (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+.\" SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+.\" POSSIBILITY OF SUCH DAMAGE.
+.\"
+.Dd January 1, 2020
+.Dt AQ 4
+.Os
+.Sh NAME
+.Nm aq
+.Nd Aquantia AQC multigigabit Network driver
+.Sh SYNOPSIS
+.Cd "aq* at pci? dev ? function ?"
+.Sh DESCRIPTION
+The
+.Nm
+driver supports Aquantia AQC series controllers.
+Supported controllers include:
+.Pp
+.Bl -bullet -compact
+.It
+AQC107 10 Gigabit Network Adapter
+.It
+AQC108 5 Gigabit Network Adapter
+.It
+AQC109 2.5 Gigabit Network Adapter
+.It
+AQC111 5 Gigabit Network Adapter
+.It
+AQC112 2.5 Gigabit Network Adapter
+.It
+AQC107S 10 Gigabit Network Adapter
+.It
+AQC108S 5 Gigabit Network Adapter
+.It
+AQC109S 2.5 Gigabit Network Adapter
+.It
+AQC111S 5 Gigabit Network Adapter
+.It
+AQC112S 2.5 Gigabit Network Adapter
+.It
+D107 10 Gigabit Network Adapter
+.It
+D108 5 Gigabit Network Adapter
+.It
+D109 2.5 Gigabit Network Adapter
+.El
+.Sh SEE ALSO
+.Xr arp 4 ,
+.Xr ifmedia 4 ,
+.Xr netintro 4 ,
+.Xr pci 4 ,
+.Xr ifconfig 8
+.Sh HISTORY
+The
+.Nm
+driver first appeared in
+.Nx 10.0 , and is based on the
+.Fx
+driver of the same name, but has been drastically rewritten by Ryo Shimizu.
+.Sh BUGS
+.Nm
+driver does not support the VLAN hardware filter.
Index: src/sys/dev/pci/if_aq.c
diff -u /dev/null src/sys/dev/pci/if_aq.c:1.1
--- /dev/null Wed Jan 1 10:11:22 2020
+++ src/sys/dev/pci/if_aq.c Wed Jan 1 10:11:21 2020
@@ -0,0 +1,4582 @@
+/* $NetBSD: if_aq.c,v 1.1 2020/01/01 10:11:21 ryo Exp $ */
+
+/**
+ * aQuantia Corporation Network Driver
+ * Copyright (C) 2014-2017 aQuantia Corporation. All rights reserved
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * (1) Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * (2) Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials provided
+ * with the distribution.
+ *
+ * (3) The name of the author may not be used to endorse or promote
+ * products derived from this software without specific prior
+ * written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+ * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+/*-
+ * Copyright (c) 2020 Ryo Shimizu <r...@nerv.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__KERNEL_RCSID(0, "$NetBSD: if_aq.c,v 1.1 2020/01/01 10:11:21 ryo Exp $");
+
+#ifdef _KERNEL_OPT
+#include "opt_if_aq.h"
+#endif
+
+#include <sys/param.h>
+#include <sys/types.h>
+#include <sys/bitops.h>
+#include <sys/cprng.h>
+#include <sys/cpu.h>
+#include <sys/interrupt.h>
+#include <sys/module.h>
+#include <sys/pcq.h>
+
+#include <net/bpf.h>
+#include <net/if.h>
+#include <net/if_dl.h>
+#include <net/if_media.h>
+#include <net/if_ether.h>
+#include <net/rss_config.h>
+
+#include <dev/pci/pcivar.h>
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcidevs.h>
+
+/* driver configuration */
+#define CONFIG_INTR_MODERATION_ENABLE true /* delayed interrupt */
+#undef CONFIG_LRO_SUPPORT /* no LRO not suppoted */
+#undef CONFIG_NO_TXRX_INDEPENDENT /* share TX/RX interrupts */
+
+#define AQ_NINTR_MAX (AQ_RSSQUEUE_MAX + AQ_RSSQUEUE_MAX + 1)
+ /* TX + RX + LINK. must be <= 32 */
+#define AQ_LINKSTAT_IRQ 31 /* for legacy mode */
+
+#define AQ_TXD_NUM 2048 /* per ring. 8*n && 32~8184 */
+#define AQ_RXD_NUM 2048 /* per ring. 8*n && 32~8184 */
+/* minimum required to send a packet (vlan needs additional TX descriptor) */
+#define AQ_TXD_MIN (1 + 1)
+
+
+/* hardware specification */
+#define AQ_RINGS_NUM 32
+#define AQ_RSSQUEUE_MAX 8
+#define AQ_RX_DESCRIPTOR_MIN 32
+#define AQ_TX_DESCRIPTOR_MIN 32
+#define AQ_RX_DESCRIPTOR_MAX 8184
+#define AQ_TX_DESCRIPTOR_MAX 8184
+#define AQ_TRAFFICCLASS_NUM 8
+#define AQ_RSS_HASHKEY_SIZE 40
+#define AQ_RSS_INDIRECTION_TABLE_MAX 64
+
+/*
+ * TERMINOLOGY
+ * MPI = MAC PHY INTERFACE?
+ * RPO = RX Protocol Offloading
+ * TPO = TX Protocol Offloading
+ * RPF = RX Packet Filter
+ * TPB = TX Packet buffer
+ * RPB = RX Packet buffer
+ */
+
+/* registers */
+#define AQ_FW_SOFTRESET_REG 0x0000
+#define AQ_FW_SOFTRESET_RESET __BIT(15) /* soft reset bit */
+#define AQ_FW_SOFTRESET_DIS __BIT(14) /* reset disable */
+
+#define AQ_FW_VERSION_REG 0x0018
+#define AQ_HW_REVISION_REG 0x001c
+#define AQ_GLB_NVR_INTERFACE1_REG 0x0100
+
+#define AQ_FW_MBOX_CMD_REG 0x0200
+#define AQ_FW_MBOX_CMD_EXECUTE 0x00008000
+#define AQ_FW_MBOX_CMD_BUSY 0x00000100
+#define AQ_FW_MBOX_ADDR_REG 0x0208
+#define AQ_FW_MBOX_VAL_REG 0x020c
+
+#define FW2X_LED_MIN_VERSION 0x03010026 /* >= 3.1.38 */
+#define FW2X_LED_REG 0x031c
+#define FW2X_LED_DEFAULT 0x00000000
+#define FW2X_LED_NONE 0x0000003f
+#define FW2X_LINKLED __BITS(0,1)
+#define FW2X_LINKLED_ACTIVE 0
+#define FW2X_LINKLED_ON 1
+#define FW2X_LINKLED_BLINK 2
+#define FW2X_LINKLED_OFF 3
+#define FW2X_STATUSLED __BITS(2,5)
+#define FW2X_STATUSLED_ORANGE 0
+#define FW2X_STATUSLED_ORANGE_BLINK 2
+#define FW2X_STATUSLED_OFF 3
+#define FW2X_STATUSLED_GREEN 4
+#define FW2X_STATUSLED_ORANGE_GREEN_BLINK 8
+#define FW2X_STATUSLED_GREEN_BLINK 10
+
+#define FW_MPI_MBOX_ADDR_REG 0x0360
+#define FW1X_MPI_INIT1_REG 0x0364
+#define FW1X_MPI_CONTROL_REG 0x0368
+#define FW1X_MPI_STATE_REG 0x036c
+#define FW1X_MPI_STATE_MODE __BITS(7,0)
+#define FW1X_MPI_STATE_SPEED __BITS(32,16)
+#define FW1X_MPI_STATE_DISABLE_DIRTYWAKE __BITS(25)
+#define FW1X_MPI_STATE_DOWNSHIFT __BITS(31,28)
+#define FW1X_MPI_INIT2_REG 0x0370
+#define FW1X_MPI_EFUSEADDR_REG 0x0374
+
+#define FW2X_MPI_EFUSEADDR_REG 0x0364
+#define FW2X_MPI_CONTROL_REG 0x0368 /* 64bit */
+#define FW2X_MPI_STATE_REG 0x0370 /* 64bit */
+#define FW_BOOT_EXIT_CODE_REG 0x0388
+#define RBL_STATUS_DEAD 0x0000dead
+#define RBL_STATUS_SUCCESS 0x0000abba
+#define RBL_STATUS_FAILURE 0x00000bad
+#define RBL_STATUS_HOST_BOOT 0x0000f1a7
+
+#define AQ_FW_GLB_CPU_SEM_REG(i) (0x03a0 + (i) * 4)
+#define AQ_FW_SEM_RAM_REG AQ_FW_GLB_CPU_SEM_REG(2)
+
+#define AQ_FW_GLB_CTL2_REG 0x0404
+#define AQ_FW_GLB_CTL2_MCP_UP_FORCE_INTERRUPT __BIT(1)
+
+#define AQ_GLB_GENERAL_PROVISIONING9_REG 0x0520
+#define AQ_GLB_NVR_PROVISIONING2_REG 0x0534
+
+#define FW_MPI_DAISY_CHAIN_STATUS_REG 0x0704
+
+#define AQ_PCI_REG_CONTROL_6_REG 0x1014
+
+// msix bitmap */
+#define AQ_INTR_STATUS_REG 0x2000 /* intr status */
+#define AQ_INTR_STATUS_CLR_REG 0x2050 /* intr status clear */
+#define AQ_INTR_MASK_REG 0x2060 /* intr mask set */
+#define AQ_INTR_MASK_CLR_REG 0x2070 /* intr mask clear */
+#define AQ_INTR_AUTOMASK_REG 0x2090
+
+/* AQ_INTR_IRQ_MAP_TXRX_REG[AQ_RINGS_NUM] 0x2100-0x2140 */
+#define AQ_INTR_IRQ_MAP_TXRX_REG(i) (0x2100 + ((i) / 2) * 4)
+#define AQ_INTR_IRQ_MAP_TX_REG(i) AQ_INTR_IRQ_MAP_TXRX_REG(i)
+#define AQ_INTR_IRQ_MAP_TX_IRQMAP(i) (__BITS(28,24) >> (((i) & 1)*8))
+#define AQ_INTR_IRQ_MAP_TX_EN(i) (__BIT(31) >> (((i) & 1)*8))
+#define AQ_INTR_IRQ_MAP_RX_REG(i) AQ_INTR_IRQ_MAP_TXRX_REG(i)
+#define AQ_INTR_IRQ_MAP_RX_IRQMAP(i) (__BITS(12,8) >> (((i) & 1)*8))
+#define AQ_INTR_IRQ_MAP_RX_EN(i) (__BIT(15) >> (((i) & 1)*8))
+
+/* AQ_GEN_INTR_MAP_REG[AQ_RINGS_NUM] 0x2180-0x2200 */
+#define AQ_GEN_INTR_MAP_REG(i) (0x2180 + (i) * 4)
+#define AQ_B0_ERR_INT 8
+
+#define AQ_INTR_CTRL_REG 0x2300
+#define AQ_INTR_CTRL_IRQMODE __BITS(1,0)
+#define AQ_INTR_CTRL_IRQMODE_LEGACY 0
+#define AQ_INTR_CTRL_IRQMODE_MSI 1
+#define AQ_INTR_CTRL_IRQMODE_MSIX 2
+#define AQ_INTR_CTRL_MULTIVEC __BIT(2)
+#define AQ_INTR_CTRL_AUTO_MASK __BIT(5)
+#define AQ_INTR_CTRL_CLR_ON_READ __BIT(7)
+#define AQ_INTR_CTRL_RESET_DIS __BIT(29)
+#define AQ_INTR_CTRL_RESET_IRQ __BIT(31)
+
+#define AQ_MBOXIF_POWER_GATING_CONTROL_REG 0x32a8
+
+#define FW_MPI_RESETCTRL_REG 0x4000
+#define FW_MPI_RESETCTRL_RESET_DIS __BIT(29)
+
+#define RX_SYSCONTROL_REG 0x5000
+#define RX_SYSCONTROL_RPB_DMA_LOOPBACK __BIT(6)
+#define RX_SYSCONTROL_RPF_TPO_LOOPBACK __BIT(8)
+#define RX_SYSCONTROL_RESET_DIS __BIT(29)
+
+#define RX_TCP_RSS_HASH_REG 0x5040
+#define RX_TCP_RSS_HASH_RPF2 __BITS(19,16)
+#define RX_TCP_RSS_HASH_TYPE __BITS(15,0)
+
+/* for RPF_*_REG.ACTION */
+#define RPF_ACTION_DISCARD 0
+#define RPF_ACTION_HOST 1
+#define RPF_ACTION_MANAGEMENT 2
+#define RPF_ACTION_HOST_MANAGEMENT 3
+#define RPF_ACTION_WOL 4
+
+#define RPF_L2BC_REG 0x5100
+#define RPF_L2BC_EN __BIT(0)
+#define RPF_L2BC_PROMISC __BIT(3)
+#define RPF_L2BC_ACTION __BITS(12,14)
+#define RPF_L2BC_THRESHOLD __BITS(31,16)
+
+/* RPF_L2UC_*_REG[34] (actual [38]?) */
+#define RPF_L2UC_LSW_REG(i) (0x5110 + (i) * 8)
+#define RPF_L2UC_MSW_REG(i) (0x5114 + (i) * 8)
+#define RPF_L2UC_MSW_MACADDR_HI __BITS(15,0)
+#define RPF_L2UC_MSW_ACTION __BITS(18,16)
+#define RPF_L2UC_MSW_EN __BIT(31)
+#define AQ_HW_MAC_OWN 0 /* index of own address */
+#define AQ_HW_MAC_NUM 34
+
+/* RPF_MCAST_FILTER_REG[12] 0x5250-0x5280 */
+#define RPF_MCAST_FILTER_REG(i) (0x5250 + (i) * 4)
+#define RPF_MCAST_FILTER_EN __BIT(31)
+#define RPF_MCAST_FILTER_MASK_REG 0x5270
+#define RPF_MCAST_FILTER_MASK_ALLMULTI __BIT(14)
+
+#define RPF_VLAN_MODE_REG 0x5280
+#define RPF_VLAN_MODE_PROMISC __BIT(1)
+#define RPF_VLAN_MODE_ACCEPT_UNTAGGED __BIT(2)
+#define RPF_VLAN_MODE_UNTAGGED_ACTION __BITS(5,3)
+
+#define RPF_VLAN_TPID_REG 0x5284
+#define RPF_VLAN_TPID_OUTER __BITS(31,16)
+#define RPF_VLAN_TPID_INNER __BITS(15,0)
+
+/* RPF_VLAN_FILTER_REG[16] 0x5290-0x52d0 */
+#define RPF_VLAN_MAX_FILTERS 16
+#define RPF_VLAN_FILTER_REG(i) (0x5290 + (i) * 4)
+#define RPF_VLAN_FILTER_EN __BIT(31)
+#define RPF_VLAN_FILTER_RXQ_EN __BIT(28)
+#define RPF_VLAN_FILTER_RXQ __BITS(24,20)
+#define RPF_VLAN_FILTER_ACTION __BITS(18,16)
+#define RPF_VLAN_FILTER_ID __BITS(11,0)
+
+/* RPF_ETHERTYPE_FILTER_REG[AQ_RINGS_NUM] 0x5300-0x5380 */
+#define RPF_ETHERTYPE_FILTER_REG(i) (0x5300 + (i) * 4)
+#define RPF_ETHERTYPE_FILTER_EN __BIT(31)
+#define RPF_ETHERTYPE_FILTER_PRIO_EN __BIT(30)
+#define RPF_ETHERTYPE_FILTER_RXQF_EN __BIT(29)
+#define RPF_ETHERTYPE_FILTER_PRIO __BITS(28,26)
+#define RPF_ETHERTYPE_FILTER_RXQF __BITS(24,20)
+#define RPF_ETHERTYPE_FILTER_MNG_RXQF __BIT(19)
+#define RPF_ETHERTYPE_FILTER_ACTION __BITS(18,16)
+#define RPF_ETHERTYPE_FILTER_VAL __BITS(15,0)
+
+/* RPF_L3_FILTER_REG[8] 0x5380-0x53a0 */
+#define RPF_L3_FILTER_REG(i) (0x5380 + (i) * 4)
+#define RPF_L3_FILTER_L4_EN __BIT(31)
+#define RPF_L3_FILTER_IPV6_EN __BIT(30)
+#define RPF_L3_FILTER_SRCADDR_EN __BIT(29)
+#define RPF_L3_FILTER_DSTADDR_EN __BIT(28)
+#define RPF_L3_FILTER_L4_SRCPORT_EN __BIT(27)
+#define RPF_L3_FILTER_L4_DSTPORT_EN __BIT(26)
+#define RPF_L3_FILTER_L4_PROTO_EN __BIT(25)
+#define RPF_L3_FILTER_ARP_EN __BIT(24)
+#define RPF_L3_FILTER_L4_RXQUEUE_EN __BIT(23)
+#define RPF_L3_FILTER_L4_RXQUEUE_MANAGEMENT_EN __BIT(22)
+#define RPF_L3_FILTER_L4_ACTION __BITS(16,18)
+#define RPF_L3_FILTER_L4_RXQUEUE __BITS(12,8)
+#define RPF_L3_FILTER_L4_PROTO __BITS(2,0)
+#define RPF_L3_FILTER_L4_PROTO_TCP 0
+#define RPF_L3_FILTER_L4_PROTO_UDP 1
+#define RPF_L3_FILTER_L4_PROTO_SCTP 2
+#define RPF_L3_FILTER_L4_PROTO_ICMP 3
+/* parameters of RPF_L3_FILTER_REG[8] */
+#define RPF_L3_FILTER_SRCADDR_REG(i) (0x53b0 + (i) * 4)
+#define RPF_L3_FILTER_DSTADDR_REG(i) (0x53d0 + (i) * 4)
+#define RPF_L3_FILTER_L4_SRCPORT_REG(i) (0x5400 + (i) * 4)
+#define RPF_L3_FILTER_L4_DSTPORT_REG(i) (0x5420 + (i) * 4)
+
+#define RX_FLR_RSS_CONTROL1_REG 0x54c0
+#define RX_FLR_RSS_CONTROL1_EN __BIT(31)
+
+#define RPF_RPB_RX_TC_UPT_REG 0x54c4
+#define RPF_RPB_RX_TC_UPT_MASK(i) (0x00000007 << ((i) * 4))
+
+#define RPF_RSS_KEY_ADDR_REG 0x54d0
+#define RPF_RSS_KEY_ADDR __BITS(4,0)
+#define RPF_RSS_KEY_WR_EN __BIT(5)
+#define RPF_RSS_KEY_WR_DATA_REG 0x54d4
+#define RPF_RSS_KEY_RD_DATA_REG 0x54d8
+
+#define RPF_RSS_REDIR_ADDR_REG 0x54e0
+#define RPF_RSS_REDIR_ADDR __BITS(3,0)
+#define RPF_RSS_REDIR_WR_EN __BIT(4)
+
+#define RPF_RSS_REDIR_WR_DATA_REG 0x54e4
+#define RPF_RSS_REDIR_WR_DATA __BITS(15,0)
+
+#define RPO_HWCSUM_REG 0x5580
+#define RPO_HWCSUM_IP4CSUM_EN __BIT(1)
+#define RPO_HWCSUM_L4CSUM_EN __BIT(0) /* TCP/UDP/SCTP */
+
+#define RPO_LRO_ENABLE_REG 0x5590
+
+#define RPO_LRO_CONF_REG 0x5594
+#define RPO_LRO_CONF_QSESSION_LIMIT __BITS(13,12)
+#define RPO_LRO_CONF_TOTAL_DESC_LIMIT __BITS(6,5)
+#define RPO_LRO_CONF_PATCHOPTIMIZATION_EN __BIT(15)
+#define RPO_LRO_CONF_MIN_PAYLOAD_OF_FIRST_PKT __BITS(4,0)
+#define RPO_LRO_RSC_MAX_REG 0x5598
+
+/* RPO_LRO_LDES_MAX_REG[32/8] 0x55a0-0x55b0 */
+#define RPO_LRO_LDES_MAX_REG(i) (0x55a0 + (i / 8) * 4)
+#define RPO_LRO_LDES_MAX_MASK(i) (0x00000003 << ((i & 7) * 4))
+#define RPO_LRO_TB_DIV_REG 0x5620
+#define RPO_LRO_TB_DIV __BITS(20,31)
+#define RPO_LRO_INACTIVE_IVAL_REG 0x5620
+#define RPO_LRO_INACTIVE_IVAL __BITS(10,19)
+#define RPO_LRO_MAX_COALESCING_IVAL_REG 0x5620
+#define RPO_LRO_MAX_COALESCING_IVAL __BITS(9,0)
+
+#define RPB_RPF_RX_REG 0x5700
+#define RPB_RPF_RX_TC_MODE __BIT(8)
+#define RPB_RPF_RX_FC_MODE __BITS(5,4)
+#define RPB_RPF_RX_BUF_EN __BIT(0)
+
+/* RPB_RXB_BUFSIZE_REG[AQ_TRAFFICCLASS_NUM] 0x5710-0x5790 */
+#define RPB_RXB_BUFSIZE_REG(i) (0x5710 + (i) * 0x10)
+#define RPB_RXB_BUFSIZE __BITS(8,0)
+#define RPB_RXB_XOFF_REG(i) (0x5714 + (i) * 0x10)
+#define RPB_RXB_XOFF_EN __BIT(31)
+#define RPB_RXB_XOFF_THRESH_HI __BITS(29,16)
+#define RPB_RXB_XOFF_THRESH_LO __BITS(13,0)
+
+#define RX_DMA_DESC_CACHE_INIT_REG 0x5a00
+#define RX_DMA_DESC_CACHE_INIT __BIT(0)
+
+#define RX_DMA_INT_DESC_WRWB_EN_REG 0x05a30
+#define RX_DMA_INT_DESC_WRWB_EN __BIT(2)
+#define RX_DMA_INT_DESC_MODERATE_EN __BIT(3)
+
+/* RX_INTR_MODERATION_CTL_REG[AQ_RINGS_NUM] 0x5a40-0x5ac0 */
+#define RX_INTR_MODERATION_CTL_REG(i) (0x5a40 + (i) * 4)
+#define RX_INTR_MODERATION_CTL_EN __BIT(1)
+#define RX_INTR_MODERATION_CTL_MIN __BITS(15,8)
+#define RX_INTR_MODERATION_CTL_MAX __BITS(24,16)
+
+/* RX_DMA_DESC_*[AQ_RINGS_NUM] 0x5b00-0x5f00 */
+#define RX_DMA_DESC_BASE_ADDRLSW_REG(i) (0x5b00 + (i) * 0x20)
+#define RX_DMA_DESC_BASE_ADDRMSW_REG(i) (0x5b04 + (i) * 0x20)
+#define RX_DMA_DESC_REG(i) (0x5b08 + (i) * 0x20)
+#define RX_DMA_DESC_LEN __BITS(12,3) /* RXD_NUM/8 */
+#define RX_DMA_DESC_RESET __BIT(25)
+#define RX_DMA_DESC_HEADER_SPLIT __BIT(28)
+#define RX_DMA_DESC_VLAN_STRIP __BIT(29)
+#define RX_DMA_DESC_EN __BIT(31)
+#define RX_DMA_DESC_HEAD_PTR_REG(i) (0x5b0c + (i) * 0x20)
+#define RX_DMA_DESC_HEAD_PTR __BITS(12,0)
+#define RX_DMA_DESC_TAIL_PTR_REG(i) (0x5b10 + (i) * 0x20)
+#define RX_DMA_DESC_BUFSIZE_REG(i) (0x5b18 + (i) * 0x20)
+#define RX_DMA_DESC_BUFSIZE_DATA __BITS(4,0)
+#define RX_DMA_DESC_BUFSIZE_HDR __BITS(12,8)
+
+/* RX_DMA_DCAD_REG[AQ_RINGS_NUM] 0x6100-0x6180 */
+#define RX_DMA_DCAD_REG(i) (0x6100 + (i) * 4)
+#define RX_DMA_DCAD_CPUID __BITS(7,0)
+#define RX_DMA_DCAD_PAYLOAD_EN __BIT(29)
+#define RX_DMA_DCAD_HEADER_EN __BIT(30)
+#define RX_DMA_DCAD_DESC_EN __BIT(31)
+
+#define RX_DMA_DCA_REG 0x6180
+#define RX_DMA_DCA_EN __BIT(31)
+#define RX_DMA_DCA_MODE __BITS(3,0)
+
+/* counters */
+#define RX_DMA_GOOD_PKT_COUNTERLSW 0x6800
+#define RX_DMA_GOOD_OCTET_COUNTERLSW 0x6808
+#define RX_DMA_DROP_PKT_CNT_REG 0x6818
+#define RX_DMA_COALESCED_PKT_CNT_REG 0x6820
+
+#define TX_SYSCONTROL_REG 0x7000
+#define TX_SYSCONTROL_TPB_DMA_LOOPBACK __BIT(6)
+#define TX_SYSCONTROL_TPO_PKT_LOOPBACK __BIT(7)
+#define TX_SYSCONTROL_RESET_DIS __BIT(29)
+
+#define TX_TPO2_REG 0x7040
+#define TX_TPO2_EN __BIT(16)
+
+#define TPS_DESC_VM_ARB_MODE_REG 0x7300
+#define TPS_DESC_VM_ARB_MODE __BIT(0)
+#define TPS_DESC_RATE_REG 0x7310
+#define TPS_DESC_RATE_TA_RST __BIT(31)
+#define TPS_DESC_RATE_LIM __BITS(10,0)
+#define TPS_DESC_TC_ARB_MODE_REG 0x7200
+#define TPS_DESC_TC_ARB_MODE __BITS(1,0)
+#define TPS_DATA_TC_ARB_MODE_REG 0x7100
+#define TPS_DATA_TC_ARB_MODE __BIT(0)
+
+/* TPS_DATA_TCT_REG[AQ_TRAFFICCLASS_NUM] 0x7110-0x7130 */
+#define TPS_DATA_TCT_REG(i) (0x7110 + (i) * 4)
+#define TPS_DATA_TCT_CREDIT_MAX __BITS(16,27)
+#define TPS_DATA_TCT_WEIGHT __BITS(8,0)
+/* TPS_DATA_TCT_REG[AQ_TRAFFICCLASS_NUM] 0x7210-0x7230 */
+#define TPS_DESC_TCT_REG(i) (0x7210 + (i) * 4)
+#define TPS_DESC_TCT_CREDIT_MAX __BITS(16,27)
+#define TPS_DESC_TCT_WEIGHT __BITS(8,0)
+
+#define AQ_HW_TXBUF_MAX 160
+#define AQ_HW_RXBUF_MAX 320
+
+#define TPO_HWCSUM_REG 0x7800
+#define TPO_HWCSUM_IP4CSUM_EN __BIT(1)
+#define TPO_HWCSUM_L4CSUM_EN __BIT(0) /* TCP/UDP/SCTP */
+
+#define TDM_LSO_EN_REG 0x7810
+
+#define THM_LSO_TCP_FLAG1_REG 0x7820
+#define THM_LSO_TCP_FLAG1_FIRST __BITS(11,0)
+#define THM_LSO_TCP_FLAG1_MID __BITS(27,16)
+#define THM_LSO_TCP_FLAG2_REG 0x7824
+#define THM_LSO_TCP_FLAG2_LAST __BITS(11,0)
+
+#define TPB_TX_BUF_REG 0x7900
+#define TPB_TX_BUF_EN __BIT(0)
+#define TPB_TX_BUF_SCP_INS_EN __BIT(2)
+#define TPB_TX_BUF_TC_MODE_EN __BIT(8)
+
+/* TPB_TXB_BUFSIZE_REG[AQ_TRAFFICCLASS_NUM] 0x7910-7990 */
+#define TPB_TXB_BUFSIZE_REG(i) (0x7910 + (i) * 0x10)
+#define TPB_TXB_BUFSIZE __BITS(7,0)
+#define TPB_TXB_THRESH_REG(i) (0x7914 + (i) * 0x10)
+#define TPB_TXB_THRESH_HI __BITS(16,28)
+#define TPB_TXB_THRESH_LO __BITS(12,0)
+
+#define AQ_HW_TX_DMA_TOTAL_REQ_LIMIT_REG 0x7b20
+#define TX_DMA_INT_DESC_WRWB_EN_REG 0x7b40
+#define TX_DMA_INT_DESC_WRWB_EN __BIT(1)
+#define TX_DMA_INT_DESC_MODERATE_EN __BIT(4)
+
+/* TX_DMA_DESC_*[AQ_RINGS_NUM] 0x7c00-0x8400 */
+#define TX_DMA_DESC_BASE_ADDRLSW_REG(i) (0x7c00 + (i) * 0x40)
+#define TX_DMA_DESC_BASE_ADDRMSW_REG(i) (0x7c04 + (i) * 0x40)
+#define TX_DMA_DESC_REG(i) (0x7c08 + (i) * 0x40)
+#define TX_DMA_DESC_LEN __BITS(12, 3) /* TXD_NUM/8 */
+#define TX_DMA_DESC_EN __BIT(31)
+#define TX_DMA_DESC_HEAD_PTR_REG(i) (0x7c0c + (i) * 0x40)
+#define TX_DMA_DESC_HEAD_PTR __BITS(12,0)
+#define TX_DMA_DESC_TAIL_PTR_REG(i) (0x7c10 + (i) * 0x40)
+#define TX_DMA_DESC_WRWB_THRESH_REG(i) (0x7c18 + (i) * 0x40)
+#define TX_DMA_DESC_WRWB_THRESH __BITS(14,8)
+
+/* TDM_DCAD_REG[AQ_RINGS_NUM] 0x8400-0x8480 */
+#define TDM_DCAD_REG(i) (0x8400 + (i) * 4)
+#define TDM_DCAD_CPUID __BITS(7,0)
+#define TDM_DCAD_CPUID_EN __BIT(31)
+
+#define TDM_DCA_REG 0x8480
+#define TDM_DCA_EN __BIT(31)
+#define TDM_DCA_MODE __BITS(3,0)
+
+/* TX_INTR_MODERATION_CTL_REG[AQ_RINGS_NUM] 0x8980-0x8a00 */
+#define TX_INTR_MODERATION_CTL_REG(i) (0x8980 + (i) * 4)
+#define TX_INTR_MODERATION_CTL_EN __BIT(1)
+#define TX_INTR_MODERATION_CTL_MIN __BITS(15,8)
+#define TX_INTR_MODERATION_CTL_MAX __BITS(24,16)
+
+#define FW1X_CTRL_10G __BIT(0)
+#define FW1X_CTRL_5G __BIT(1)
+#define FW1X_CTRL_5GSR __BIT(2)
+#define FW1X_CTRL_2G5 __BIT(3)
+#define FW1X_CTRL_1G __BIT(4)
+#define FW1X_CTRL_100M __BIT(5)
+
+#define FW2X_CTRL_10BASET_HD __BIT(0)
+#define FW2X_CTRL_10BASET_FD __BIT(1)
+#define FW2X_CTRL_100BASETX_HD __BIT(2)
+#define FW2X_CTRL_100BASET4_HD __BIT(3)
+#define FW2X_CTRL_100BASET2_HD __BIT(4)
+#define FW2X_CTRL_100BASETX_FD __BIT(5)
+#define FW2X_CTRL_100BASET2_FD __BIT(6)
+#define FW2X_CTRL_1000BASET_HD __BIT(7)
+#define FW2X_CTRL_1000BASET_FD __BIT(8)
+#define FW2X_CTRL_2P5GBASET_FD __BIT(9)
+#define FW2X_CTRL_5GBASET_FD __BIT(10)
+#define FW2X_CTRL_10GBASET_FD __BIT(11)
+#define FW2X_CTRL_RESERVED1 __BIT(32)
+#define FW2X_CTRL_10BASET_EEE __BIT(33)
+#define FW2X_CTRL_RESERVED2 __BIT(34)
+#define FW2X_CTRL_PAUSE __BIT(35)
+#define FW2X_CTRL_ASYMMETRIC_PAUSE __BIT(36)
+#define FW2X_CTRL_100BASETX_EEE __BIT(37)
+#define FW2X_CTRL_RESERVED3 __BIT(38)
+#define FW2X_CTRL_RESERVED4 __BIT(39)
+#define FW2X_CTRL_1000BASET_FD_EEE __BIT(40)
+#define FW2X_CTRL_2P5GBASET_FD_EEE __BIT(41)
+#define FW2X_CTRL_5GBASET_FD_EEE __BIT(42)
+#define FW2X_CTRL_10GBASET_FD_EEE __BIT(43)
+#define FW2X_CTRL_RESERVED5 __BIT(44)
+#define FW2X_CTRL_RESERVED6 __BIT(45)
+#define FW2X_CTRL_RESERVED7 __BIT(46)
+#define FW2X_CTRL_RESERVED8 __BIT(47)
+#define FW2X_CTRL_RESERVED9 __BIT(48)
+#define FW2X_CTRL_CABLE_DIAG __BIT(49)
+#define FW2X_CTRL_TEMPERATURE __BIT(50)
+#define FW2X_CTRL_DOWNSHIFT __BIT(51)
+#define FW2X_CTRL_PTP_AVB_EN __BIT(52)
+#define FW2X_CTRL_MEDIA_DETECT __BIT(53)
+#define FW2X_CTRL_LINK_DROP __BIT(54)
+#define FW2X_CTRL_SLEEP_PROXY __BIT(55)
+#define FW2X_CTRL_WOL __BIT(56)
+#define FW2X_CTRL_MAC_STOP __BIT(57)
+#define FW2X_CTRL_EXT_LOOPBACK __BIT(58)
+#define FW2X_CTRL_INT_LOOPBACK __BIT(59)
+#define FW2X_CTRL_EFUSE_AGENT __BIT(60)
+#define FW2X_CTRL_WOL_TIMER __BIT(61)
+#define FW2X_CTRL_STATISTICS __BIT(62)
+#define FW2X_CTRL_TRANSACTION_ID __BIT(63)
+
+#define FW2X_SNPRINTB \
+ "\177\020" \
+ "b\x23" "PAUSE\0" \
+ "b\x24" "ASYMMETRIC-PAUSE\0" \
+ "b\x31" "CABLE-DIAG\0" \
+ "b\x32" "TEMPERATURE\0" \
+ "b\x33" "DOWNSHIFT\0" \
+ "b\x34" "PTP-AVB\0" \
+ "b\x35" "MEDIA-DETECT\0" \
+ "b\x36" "LINK-DROP\0" \
+ "b\x37" "SLEEP-PROXY\0" \
+ "b\x38" "WOL\0" \
+ "b\x39" "MAC-STOP\0" \
+ "b\x3a" "EXT-LOOPBACK\0" \
+ "b\x3b" "INT-LOOPBACK\0" \
+ "b\x3c" "EFUSE-AGENT\0" \
+ "b\x3d" "WOL-TIMER\0" \
+ "b\x3e" "STATISTICS\0" \
+ "b\x3f" "TRANSACTION-ID\0" \
+ "\0"
+
+#define FW2X_CTRL_RATE_100M FW2X_CTRL_100BASETX_FD
+#define FW2X_CTRL_RATE_1G FW2X_CTRL_1000BASET_FD
+#define FW2X_CTRL_RATE_2G5 FW2X_CTRL_2P5GBASET_FD
+#define FW2X_CTRL_RATE_5G FW2X_CTRL_5GBASET_FD
+#define FW2X_CTRL_RATE_10G FW2X_CTRL_10GBASET_FD
+#define FW2X_CTRL_RATE_MASK \
+ (FW2X_CTRL_RATE_100M | \
+ FW2X_CTRL_RATE_1G | \
+ FW2X_CTRL_RATE_2G5 | \
+ FW2X_CTRL_RATE_5G | \
+ FW2X_CTRL_RATE_10G)
+#define FW2X_CTRL_EEE_MASK \
+ (FW2X_CTRL_10BASET_EEE | \
+ FW2X_CTRL_100BASETX_EEE | \
+ FW2X_CTRL_1000BASET_FD_EEE | \
+ FW2X_CTRL_2P5GBASET_FD_EEE | \
+ FW2X_CTRL_5GBASET_FD_EEE | \
+ FW2X_CTRL_10GBASET_FD_EEE)
+
+typedef enum aq_fw_bootloader_mode {
+ FW_BOOT_MODE_UNKNOWN = 0,
+ FW_BOOT_MODE_FLB,
+ FW_BOOT_MODE_RBL_FLASH,
+ FW_BOOT_MODE_RBL_HOST_BOOTLOAD
+} aq_fw_bootloader_mode_t;
+
+#define AQ_WRITE_REG(sc, reg, val) \
+ bus_space_write_4((sc)->sc_iot, (sc)->sc_ioh, (reg), (val))
+
+#define AQ_READ_REG(sc, reg) \
+ bus_space_read_4((sc)->sc_iot, (sc)->sc_ioh, (reg))
+
+#define AQ_READ64_REG(sc, reg) \
+ ((uint64_t)AQ_READ_REG(sc, reg) | \
+ (((uint64_t)AQ_READ_REG(sc, (reg) + 4)) << 32))
+
+#define AQ_WRITE64_REG(sc, reg, val) \
+ do { \
+ AQ_WRITE_REG(sc, reg, (uint32_t)val); \
+ AQ_WRITE_REG(sc, reg + 4, (uint32_t)(val >> 32)); \
+ } while (/* CONSTCOND */0)
+
+#define AQ_READ_REG_BIT(sc, reg, mask) \
+ __SHIFTOUT(AQ_READ_REG(sc, reg), mask)
+
+#define AQ_WRITE_REG_BIT(sc, reg, mask, val) \
+ do { \
+ uint32_t _v; \
+ _v = AQ_READ_REG((sc), (reg)); \
+ _v &= ~(mask); \
+ if ((val) != 0) \
+ _v |= __SHIFTIN((val), (mask)); \
+ AQ_WRITE_REG((sc), (reg), _v); \
+ } while (/* CONSTCOND */ 0)
+
+#define WAIT_FOR(expr, us, n, errp) \
+ do { \
+ unsigned int _n; \
+ for (_n = n; (!(expr)) && _n != 0; --_n) { \
+ delay((us)); \
+ } \
+ if ((errp != NULL)) { \
+ if (_n == 0) \
+ *(errp) = ETIMEDOUT; \
+ else \
+ *(errp) = 0; \
+ } \
+ } while (/* CONSTCOND */ 0)
+
+#define msec_delay(x) DELAY(1000 * (x))
+
+typedef struct aq_mailbox_header {
+ uint32_t version;
+ uint32_t transaction_id;
+ int32_t error;
+} __packed aq_mailbox_header_t;
+
+typedef struct aq_hw_stats_s {
+ uint32_t uprc;
+ uint32_t mprc;
+ uint32_t bprc;
+ uint32_t erpt;
+ uint32_t uptc;
+ uint32_t mptc;
+ uint32_t bptc;
+ uint32_t erpr;
+ uint32_t mbtc;
+ uint32_t bbtc;
+ uint32_t mbrc;
+ uint32_t bbrc;
+ uint32_t ubrc;
+ uint32_t ubtc;
+ uint32_t ptc;
+ uint32_t prc;
+ uint32_t dpc; /* not exists in fw2x_msm_statistics */
+ uint32_t cprc; /* not exists in fw2x_msm_statistics */
+} __packed aq_hw_stats_s_t;
+
+typedef struct fw1x_mailbox {
+ aq_mailbox_header_t header;
+ aq_hw_stats_s_t msm;
+} __packed fw1x_mailbox_t;
+
+typedef struct fw2x_msm_statistics {
+ uint32_t uprc;
+ uint32_t mprc;
+ uint32_t bprc;
+ uint32_t erpt;
+ uint32_t uptc;
+ uint32_t mptc;
+ uint32_t bptc;
+ uint32_t erpr;
+ uint32_t mbtc;
+ uint32_t bbtc;
+ uint32_t mbrc;
+ uint32_t bbrc;
+ uint32_t ubrc;
+ uint32_t ubtc;
+ uint32_t ptc;
+ uint32_t prc;
+} __packed fw2x_msm_statistics_t;
+
+typedef struct fw2x_phy_cable_diag_data {
+ uint32_t lane_data[4];
+} __packed fw2x_phy_cable_diag_data_t;
+
+typedef struct fw2x_capabilities {
+ uint32_t caps_lo;
+ uint32_t caps_hi;
+} __packed fw2x_capabilities_t;
+
+typedef struct fw2x_mailbox { /* struct fwHostInterface */
+ aq_mailbox_header_t header;
+ fw2x_msm_statistics_t msm; /* msmStatistics_t msm; */
+ uint16_t phy_h_bit;
+ uint16_t phy_fault_code;
+ int16_t phy_temperature;
+ uint8_t cable_len;
+ uint8_t reserved1;
+ fw2x_phy_cable_diag_data_t diag_data;
+ uint32_t reserved[8];
+
+ fw2x_capabilities_t caps;
+
+ /* ... */
+} __packed fw2x_mailbox_t;
+
+typedef enum aq_link_speed {
+ AQ_LINK_NONE = 0,
+ AQ_LINK_100M = (1 << 0),
+ AQ_LINK_1G = (1 << 1),
+ AQ_LINK_2G5 = (1 << 2),
+ AQ_LINK_5G = (1 << 3),
+ AQ_LINK_10G = (1 << 4)
+} aq_link_speed_t;
+#define AQ_LINK_ALL (AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | \
+ AQ_LINK_5G | AQ_LINK_10G )
+#define AQ_LINK_AUTO AQ_LINK_ALL
+
+typedef enum aq_link_fc {
+ AQ_FC_NONE = 0,
+ AQ_FC_RX = __BIT(0),
+ AQ_FC_TX = __BIT(1),
+ AQ_FC_ALL = (AQ_FC_RX | AQ_FC_TX)
+} aq_link_fc_t;
+
+typedef enum aq_link_eee {
+ AQ_EEE_DISABLE = 0,
+ AQ_EEE_ENABLE = 1
+} aq_link_eee_t;
+
+typedef enum aq_hw_fw_mpi_state {
+ MPI_DEINIT = 0,
+ MPI_RESET = 1,
+ MPI_INIT = 2,
+ MPI_POWER = 4
+} aq_hw_fw_mpi_state_t;
+
+enum aq_media_type {
+ AQ_MEDIA_TYPE_UNKNOWN = 0,
+ AQ_MEDIA_TYPE_FIBRE,
+ AQ_MEDIA_TYPE_TP
+};
+
+struct aq_rx_desc_read {
+ uint64_t buf_addr;
+ uint64_t hdr_addr;
+} __packed;
+
+struct aq_rx_desc_wb {
+ uint32_t type;
+#define RXDESC_TYPE_RSSTYPE __BITS(3,0)
+#define RXDESC_TYPE_RSSTYPE_NONE 0
+#define RXDESC_TYPE_RSSTYPE_IPV4 2
+#define RXDESC_TYPE_RSSTYPE_IPV6 3
+#define RXDESC_TYPE_RSSTYPE_IPV4_TCP 4
+#define RXDESC_TYPE_RSSTYPE_IPV6_TCP 5
+#define RXDESC_TYPE_RSSTYPE_IPV4_UDP 6
+#define RXDESC_TYPE_RSSTYPE_IPV6_UDP 7
+#define RXDESC_TYPE_PKTTYPE_ETHER __BITS(5,4)
+#define RXDESC_TYPE_PKTTYPE_ETHER_IPV4 0
+#define RXDESC_TYPE_PKTTYPE_ETHER_IPV6 1
+#define RXDESC_TYPE_PKTTYPE_ETHER_OTHERS 2
+#define RXDESC_TYPE_PKTTYPE_ETHER_ARP 3
+#define RXDESC_TYPE_PKTTYPE_PROTO __BITS(8,6)
+#define RXDESC_TYPE_PKTTYPE_PROTO_TCP 0
+#define RXDESC_TYPE_PKTTYPE_PROTO_UDP 1
+#define RXDESC_TYPE_PKTTYPE_PROTO_SCTP 2
+#define RXDESC_TYPE_PKTTYPE_PROTO_ICMP 3
+#define RXDESC_TYPE_PKTTYPE_PROTO_OTHERS 4
+#define RXDESC_TYPE_PKTTYPE_VLAN __BIT(9)
+#define RXDESC_TYPE_PKTTYPE_VLAN_DOUBLE __BIT(10)
+#define RXDESC_TYPE_MAC_DMA_ERR __BIT(12)
+#define RXDESC_TYPE_RESERVED __BITS(18,13)
+#define RXDESC_TYPE_IPV4_CSUM_CHECKED __BIT(19) /* PKTTYPE_ETHER_IPV4 */
+#define RXDESC_TYPE_TCPUDP_CSUM_CHECKED __BIT(20)
+#define RXDESC_TYPE_SPH __BIT(21)
+#define RXDESC_TYPE_HDR_LEN __BITS(31,22)
+ uint32_t rss_hash;
+ uint16_t status;
+#define RXDESC_STATUS_DD __BIT(0)
+#define RXDESC_STATUS_EOP __BIT(1)
+#define RXDESC_STATUS_MACERR __BIT(2)
+#define RXDESC_STATUS_IPV4_CSUM_NG __BIT(3)
+#define RXDESC_STATUS_TCPUDP_CSUM_ERROR __BIT(4)
+#define RXDESC_STATUS_TCPUDP_CSUM_OK __BIT(5)
+
+#define RXDESC_STATUS_STAT __BITS(2,5)
+#define RXDESC_STATUS_ESTAT __BITS(6,11)
+#define RXDESC_STATUS_RSC_CNT __BITS(12,15)
+ uint16_t pkt_len;
+ uint16_t next_desc_ptr;
+ uint16_t vlan;
+} __packed;
+
+typedef union aq_rx_desc {
+ struct aq_rx_desc_read read;
+ struct aq_rx_desc_wb wb;
+} __packed aq_rx_desc_t;
+
+typedef struct aq_tx_desc {
+ uint64_t buf_addr;
+ uint32_t ctl1;
+#define AQ_TXDESC_CTL1_TYPE_MASK 0x00000003
+#define AQ_TXDESC_CTL1_TYPE_TXD 0x00000001
+#define AQ_TXDESC_CTL1_TYPE_TXC 0x00000002
+#define AQ_TXDESC_CTL1_BLEN __BITS(19,4) /* TXD */
+#define AQ_TXDESC_CTL1_DD __BIT(20) /* TXD */
+#define AQ_TXDESC_CTL1_EOP __BIT(21) /* TXD */
+#define AQ_TXDESC_CTL1_CMD_VLAN __BIT(22) /* TXD */
+#define AQ_TXDESC_CTL1_CMD_FCS __BIT(23) /* TXD */
+#define AQ_TXDESC_CTL1_CMD_IP4CSUM __BIT(24) /* TXD */
+#define AQ_TXDESC_CTL1_CMD_L4CSUM __BIT(25) /* TXD */
+#define AQ_TXDESC_CTL1_CMD_LSO __BIT(26) /* TXD */
+#define AQ_TXDESC_CTL1_CMD_WB __BIT(27) /* TXD */
+#define AQ_TXDESC_CTL1_CMD_VXLAN __BIT(28) /* TXD */
+#define AQ_TXDESC_CTL1_VID __BITS(15,4) /* TXC */
+#define AQ_TXDESC_CTL1_LSO_IPV6 __BIT(21) /* TXC */
+#define AQ_TXDESC_CTL1_LSO_TCP __BIT(22) /* TXC */
+ uint32_t ctl2;
+#define AQ_TXDESC_CTL2_LEN __BITS(31,14)
+#define AQ_TXDESC_CTL2_CTX_EN __BIT(13)
+#define AQ_TXDESC_CTL2_CTX_IDX __BIT(12)
+} __packed aq_tx_desc_t;
+
+struct aq_txring {
+ struct aq_softc *txr_sc;
+ int txr_index;
+ kmutex_t txr_mutex;
+ bool txr_active;
+
+ pcq_t *txr_pcq;
+ void *txr_softint;
+
+ aq_tx_desc_t *txr_txdesc; /* aq_tx_desc_t[AQ_TXD_NUM] */
+ bus_dmamap_t txr_txdesc_dmamap;
+ bus_dma_segment_t txr_txdesc_seg[1];
+ bus_size_t txr_txdesc_size;
+
+ struct {
+ struct mbuf *m;
+ bus_dmamap_t dmamap;
+ } txr_mbufs[AQ_TXD_NUM];
+ unsigned int txr_prodidx;
+ unsigned int txr_considx;
+ int txr_nfree;
+};
+
+struct aq_rxring {
+ struct aq_softc *rxr_sc;
+ int rxr_index;
+ kmutex_t rxr_mutex;
+ bool rxr_active;
+
+ aq_rx_desc_t *rxr_rxdesc; /* aq_rx_desc_t[AQ_RXD_NUM] */
+ bus_dmamap_t rxr_rxdesc_dmamap;
+ bus_dma_segment_t rxr_rxdesc_seg[1];
+ bus_size_t rxr_rxdesc_size;
+ struct {
+ struct mbuf *m;
+ bus_dmamap_t dmamap;
+ } rxr_mbufs[AQ_RXD_NUM];
+ unsigned int rxr_readidx;
+};
+
+struct aq_queue {
+ struct aq_softc *sc;
+ struct aq_txring txring;
+ struct aq_rxring rxring;
+};
+
+struct aq_softc;
+struct aq_firmware_ops {
+ int (*reset)(struct aq_softc *);
+ int (*set_mode)(struct aq_softc *, aq_hw_fw_mpi_state_t,
+ aq_link_speed_t, aq_link_fc_t, aq_link_eee_t);
+ int (*get_mode)(struct aq_softc *, aq_hw_fw_mpi_state_t *,
+ aq_link_speed_t *, aq_link_fc_t *, aq_link_eee_t *);
+ int (*get_stats)(struct aq_softc *, aq_hw_stats_s_t *);
+};
+
+#ifdef AQ_EVENT_COUNTERS
+#define AQ_EVCNT_DECL(name) \
+ char sc_evcount_##name##_name[32]; \
+ struct evcnt sc_evcount_##name##_ev;
+#define AQ_EVCNT_ATTACH(sc, name, desc, evtype) \
+ do { \
+ snprintf((sc)->sc_evcount_##name##_name, \
+ sizeof((sc)->sc_evcount_##name##_name), \
+ "%s", desc); \
+ evcnt_attach_dynamic(&(sc)->sc_evcount_##name##_ev, \
+ (evtype), NULL, device_xname((sc)->sc_dev), \
+ (sc)->sc_evcount_##name##_name); \
+ } while (/*CONSTCOND*/0)
+#define AQ_EVCNT_ATTACH_MISC(sc, name, desc) \
+ AQ_EVCNT_ATTACH(sc, name, desc, EVCNT_TYPE_MISC)
+#define AQ_EVCNT_DETACH(sc, name) \
+ evcnt_detach(&(sc)->sc_evcount_##name##_ev)
+#define AQ_EVCNT_ADD(sc, name, val) \
+ ((sc)->sc_evcount_##name##_ev.ev_count += (val))
+#endif /* AQ_EVENT_COUNTERS */
+
+#define AQ_LOCK(sc) mutex_enter(&(sc)->sc_mutex);
+#define AQ_UNLOCK(sc) mutex_exit(&(sc)->sc_mutex);
+
+struct aq_softc {
+ device_t sc_dev;
+
+ bus_space_tag_t sc_iot;
+ bus_space_handle_t sc_ioh;
+ bus_size_t sc_iosize;
+ bus_dma_tag_t sc_dmat;;
+
+ void *sc_ihs[AQ_NINTR_MAX];
+ pci_intr_handle_t *sc_intrs;
+
+ int sc_tx_irq[AQ_RSSQUEUE_MAX];
+ int sc_rx_irq[AQ_RSSQUEUE_MAX];
+ int sc_linkstat_irq;
+ bool sc_use_txrx_independent_intr;
+ bool sc_poll_linkstat;
+ bool sc_detect_linkstat;
+
+ callout_t sc_tick_ch;
+
+ int sc_nintrs;
+ bool sc_msix;
+
+ struct aq_queue sc_queue[AQ_RSSQUEUE_MAX];
+ int sc_nqueues;
+
+ pci_chipset_tag_t sc_pc;
+ pcitag_t sc_pcitag;
+ uint16_t sc_product;
+ uint16_t sc_revision;
+
+ kmutex_t sc_mutex;
+
+ struct aq_firmware_ops *sc_fw_ops;
+ uint64_t sc_fw_caps;
+ enum aq_media_type sc_media_type;
+ aq_link_speed_t sc_available_rates;
+
+ aq_link_speed_t sc_link_rate;
+ aq_link_fc_t sc_link_fc;
+ aq_link_eee_t sc_link_eee;
+
+ uint32_t sc_fw_version;
+#define FW_VERSION_MAJOR(sc) (((sc)->sc_fw_version >> 24) & 0xff)
+#define FW_VERSION_MINOR(sc) (((sc)->sc_fw_version >> 16) & 0xff)
+#define FW_VERSION_BUILD(sc) ((sc)->sc_fw_version & 0xffff)
+ uint32_t sc_features;
+#define FEATURES_MIPS 0x00000001
+#define FEATURES_TPO2 0x00000002
+#define FEATURES_RPF2 0x00000004
+#define FEATURES_MPI_AQ 0x00000008
+#define FEATURES_REV_A0 0x10000000
+#define FEATURES_REV_A (FEATURES_REV_A0)
+#define FEATURES_REV_B0 0x20000000
+#define FEATURES_REV_B1 0x40000000
+#define FEATURES_REV_B (FEATURES_REV_B0|FEATURES_REV_B1)
+ uint32_t sc_mbox_addr;
+
+ bool sc_rbl_enabled;
+ bool sc_fast_start_enabled;
+ bool sc_flash_present;
+
+ bool sc_intr_moderation_enable;
+ bool sc_rss_enable;
+
+ int sc_media_active;
+
+ struct ethercom sc_ethercom;
+ struct ether_addr sc_enaddr;
+ struct ifmedia sc_media;
+ int sc_ec_capenable; /* last ec_capenable */
+ unsigned short sc_if_flags; /* last if_flags */
+
+#ifdef AQ_EVENT_COUNTERS
+ aq_hw_stats_s_t sc_statistics[2];
+ int sc_statistics_idx;
+ bool sc_poll_statistics;
+
+ AQ_EVCNT_DECL(uprc);
+ AQ_EVCNT_DECL(mprc);
+ AQ_EVCNT_DECL(bprc);
+ AQ_EVCNT_DECL(erpt);
+ AQ_EVCNT_DECL(uptc);
+ AQ_EVCNT_DECL(mptc);
+ AQ_EVCNT_DECL(bptc);
+ AQ_EVCNT_DECL(erpr);
+ AQ_EVCNT_DECL(mbtc);
+ AQ_EVCNT_DECL(bbtc);
+ AQ_EVCNT_DECL(mbrc);
+ AQ_EVCNT_DECL(bbrc);
+ AQ_EVCNT_DECL(ubrc);
+ AQ_EVCNT_DECL(ubtc);
+ AQ_EVCNT_DECL(ptc);
+ AQ_EVCNT_DECL(prc);
+ AQ_EVCNT_DECL(dpc);
+ AQ_EVCNT_DECL(cprc);
+#endif
+};
+
+static int aq_match(device_t, cfdata_t, void *);
+static void aq_attach(device_t, device_t, void *);
+static int aq_detach(device_t, int);
+
+static int aq_setup_msix(struct aq_softc *, struct pci_attach_args *, int,
+ bool, bool);
+static int aq_setup_legacy(struct aq_softc *, struct pci_attach_args *,
+ pci_intr_type_t);
+static int aq_establish_msix_intr(struct aq_softc *, bool, bool);
+
+static int aq_ifmedia_change(struct ifnet * const);
+static void aq_ifmedia_status(struct ifnet * const, struct ifmediareq *);
+static int aq_ifflags_cb(struct ethercom *);
+static int aq_init(struct ifnet *);
+static void aq_send_common_locked(struct ifnet *, struct aq_softc *,
+ struct aq_txring *, bool);
+static int aq_transmit(struct ifnet *, struct mbuf *);
+static void aq_deferred_transmit(void *);
+static void aq_start(struct ifnet *);
+static void aq_stop(struct ifnet *, int);
+static void aq_watchdog(struct ifnet *);
+static int aq_ioctl(struct ifnet *, unsigned long, void *);
+
+static int aq_txrx_rings_alloc(struct aq_softc *);
+static void aq_txrx_rings_free(struct aq_softc *);
+static int aq_tx_pcq_alloc(struct aq_softc *, struct aq_txring *);
+static void aq_tx_pcq_free(struct aq_softc *, struct aq_txring *);
+
+static void aq_initmedia(struct aq_softc *);
+static void aq_enable_intr(struct aq_softc *, bool, bool);
+
+static void aq_tick(void *);
+static int aq_legacy_intr(void *);
+static int aq_link_intr(void *);
+static int aq_txrx_intr(void *);
+static int aq_tx_intr(void *);
+static int aq_rx_intr(void *);
+
+static int aq_set_linkmode(struct aq_softc *, aq_link_speed_t, aq_link_fc_t,
+ aq_link_eee_t);
+static int aq_get_linkmode(struct aq_softc *, aq_link_speed_t *, aq_link_fc_t *,
+ aq_link_eee_t *);
+
+static int aq_fw_reset(struct aq_softc *);
+static int aq_fw_version_init(struct aq_softc *);
+static int aq_hw_init(struct aq_softc *);
+static int aq_hw_init_ucp(struct aq_softc *);
+static int aq_hw_reset(struct aq_softc *);
+static int aq_fw_downld_dwords(struct aq_softc *, uint32_t, uint32_t *,
+ uint32_t);
+static int aq_get_mac_addr(struct aq_softc *);
+static int aq_init_rss(struct aq_softc *);
+static int aq_set_capability(struct aq_softc *);
+
+static int fw1x_reset(struct aq_softc *);
+static int fw1x_set_mode(struct aq_softc *, aq_hw_fw_mpi_state_t,
+ aq_link_speed_t, aq_link_fc_t, aq_link_eee_t);
+static int fw1x_get_mode(struct aq_softc *, aq_hw_fw_mpi_state_t *,
+ aq_link_speed_t *, aq_link_fc_t *, aq_link_eee_t *);
+static int fw1x_get_stats(struct aq_softc *, aq_hw_stats_s_t *);
+
+static int fw2x_reset(struct aq_softc *);
+static int fw2x_set_mode(struct aq_softc *, aq_hw_fw_mpi_state_t,
+ aq_link_speed_t, aq_link_fc_t, aq_link_eee_t);
+static int fw2x_get_mode(struct aq_softc *, aq_hw_fw_mpi_state_t *,
+ aq_link_speed_t *, aq_link_fc_t *, aq_link_eee_t *);
+static int fw2x_get_stats(struct aq_softc *, aq_hw_stats_s_t *);
+
+static struct aq_firmware_ops aq_fw1x_ops = {
+ .reset = fw1x_reset,
+ .set_mode = fw1x_set_mode,
+ .get_mode = fw1x_get_mode,
+ .get_stats = fw1x_get_stats
+};
+
+static struct aq_firmware_ops aq_fw2x_ops = {
+ .reset = fw2x_reset,
+ .set_mode = fw2x_set_mode,
+ .get_mode = fw2x_get_mode,
+ .get_stats = fw2x_get_stats
+};
+
+CFATTACH_DECL3_NEW(aq, sizeof(struct aq_softc),
+ aq_match, aq_attach, aq_detach, NULL, NULL, NULL, DVF_DETACH_SHUTDOWN);
+
+static const struct aq_product {
+ pci_vendor_id_t aq_vendor;
+ pci_product_id_t aq_product;
+ const char *aq_name;
+ enum aq_media_type aq_media_type;
+ aq_link_speed_t aq_available_rates;
+} aq_products[] = {
+ { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC107,
+ "Aquantia AQC107 10 Gigabit Network Adapter",
+ AQ_MEDIA_TYPE_TP, AQ_LINK_ALL
+ },
+ { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC108,
+ "Aquantia AQC108 5 Gigabit Network Adapter",
+ AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
+ },
+ { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC109,
+ "Aquantia AQC109 2.5 Gigabit Network Adapter",
+ AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
+ },
+ { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC111,
+ "Aquantia AQC111 5 Gigabit Network Adapter",
+ AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
+ },
+ { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC112,
+ "Aquantia AQC112 2.5 Gigabit Network Adapter",
+ AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
+ },
+ { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC107S,
+ "Aquantia AQC107S 10 Gigabit Network Adapter",
+ AQ_MEDIA_TYPE_TP, AQ_LINK_ALL
+ },
+ { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC108S,
+ "Aquantia AQC108S 5 Gigabit Network Adapter",
+ AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
+ },
+ { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC109S,
+ "Aquantia AQC109S 2.5 Gigabit Network Adapter",
+ AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
+ },
+ { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC111S,
+ "Aquantia AQC111S 5 Gigabit Network Adapter",
+ AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
+ },
+ { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC112S,
+ "Aquantia AQC112S 2.5 Gigabit Network Adapter",
+ AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
+ },
+ { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_D107,
+ "Aquantia D107 10 Gigabit Network Adapter",
+ AQ_MEDIA_TYPE_TP, AQ_LINK_ALL
+ },
+ { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_D108,
+ "Aquantia D108 5 Gigabit Network Adapter",
+ AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
+ },
+ { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_D109,
+ "Aquantia D109 2.5 Gigabit Network Adapter",
+ AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
+ }
+};
+
+static const struct aq_product *
+aq_lookup(const struct pci_attach_args *pa)
+{
+ unsigned int i;
+
+ for (i = 0; i < __arraycount(aq_products); i++) {
+ if (PCI_VENDOR(pa->pa_id) == aq_products[i].aq_vendor &&
+ PCI_PRODUCT(pa->pa_id) == aq_products[i].aq_product)
+ return &aq_products[i];
+ }
+ return NULL;
+}
+
+static int
+aq_match(device_t parent, cfdata_t cf, void *aux)
+{
+ struct pci_attach_args *pa = aux;
+
+ if (aq_lookup(pa) != NULL)
+ return 1;
+
+ return 0;
+}
+
+static void
+aq_attach(device_t parent, device_t self, void *aux)
+{
+ struct aq_softc *sc = device_private(self);
+ struct pci_attach_args *pa = aux;
+ struct ifnet *ifp = &sc->sc_ethercom.ec_if;
+ pci_chipset_tag_t pc;
+ pcitag_t tag;
+ pcireg_t command, memtype, bar;
+ const struct aq_product *aqp;
+ int error;
+
+ sc->sc_dev = self;
+ mutex_init(&sc->sc_mutex, MUTEX_DEFAULT, IPL_NET);
+
+ sc->sc_pc = pc = pa->pa_pc;
+ sc->sc_pcitag = tag = pa->pa_tag;
+ sc->sc_dmat = pci_dma64_available(pa) ? pa->pa_dmat64 : pa->pa_dmat;
+
+ command = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG);
+ command |= PCI_COMMAND_MASTER_ENABLE;
+ pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, command);
+
+ sc->sc_product = PCI_PRODUCT(pa->pa_id);
+ sc->sc_revision = PCI_REVISION(pa->pa_class);
+
+ aqp = aq_lookup(pa);
+ KASSERT(aqp != NULL);
+
+ pci_aprint_devinfo_fancy(pa, "Ethernet controller", aqp->aq_name, 1);
+
+ bar = pci_conf_read(pc, tag, PCI_BAR(0));
+ if ((PCI_MAPREG_MEM_ADDR(bar) == 0) ||
+ (PCI_MAPREG_TYPE(bar) != PCI_MAPREG_TYPE_MEM)) {
+ aprint_error_dev(sc->sc_dev, "wrong BAR type\n");
+ return;
+ }
+ memtype = pci_mapreg_type(pc, tag, PCI_BAR(0));
+ if (pci_mapreg_map(pa, PCI_BAR(0), memtype, 0, &sc->sc_iot, &sc->sc_ioh,
+ NULL, &sc->sc_iosize) != 0) {
+ aprint_error_dev(sc->sc_dev, "unable to map register\n");
+ return;
+ }
+
+ sc->sc_nqueues = MIN(ncpu, AQ_RSSQUEUE_MAX);
+
+ /* max queue num is 8, and must be 2^n */
+ if (ncpu >= 8)
+ sc->sc_nqueues = 8;
+ else if (ncpu >= 4)
+ sc->sc_nqueues = 4;
+ else if (ncpu >= 2)
+ sc->sc_nqueues = 2;
+ else
+ sc->sc_nqueues = 1;
+
+ int msixcount = pci_msix_count(pa->pa_pc, pa->pa_tag);
+#ifndef CONFIG_NO_TXRX_INDEPENDENT
+ if (msixcount >= (sc->sc_nqueues * 2 + 1)) {
+ /* TX intrs + RX intrs + LINKSTAT intrs */
+ sc->sc_use_txrx_independent_intr = true;
+ sc->sc_poll_linkstat = false;
+ sc->sc_msix = true;
+ } else if (msixcount >= (sc->sc_nqueues * 2)) {
+ /* TX intrs + RX intrs */
+ sc->sc_use_txrx_independent_intr = true;
+ sc->sc_poll_linkstat = true;
+ sc->sc_msix = true;
+ } else
+#endif
+ if (msixcount >= (sc->sc_nqueues + 1)) {
+ /* TX/RX intrs LINKSTAT intrs */
+ sc->sc_use_txrx_independent_intr = false;
+ sc->sc_poll_linkstat = false;
+ sc->sc_msix = true;
+ } else if (msixcount >= sc->sc_nqueues) {
+ /* TX/RX intrs */
+ sc->sc_use_txrx_independent_intr = false;
+ sc->sc_poll_linkstat = true;
+ sc->sc_msix = true;
+ } else {
+ /* giving up using MSI-X */
+ sc->sc_msix = false;
+ }
+
+ aprint_debug_dev(sc->sc_dev,
+ "ncpu=%d, pci_msix_count=%d."
+ " allocate %d interrupts for %d%s queues%s\n",
+ ncpu, msixcount,
+ (sc->sc_use_txrx_independent_intr ?
+ (sc->sc_nqueues * 2) : sc->sc_nqueues) +
+ (sc->sc_poll_linkstat ? 0 : 1),
+ sc->sc_nqueues,
+ sc->sc_use_txrx_independent_intr ? "*2" : "",
+ sc->sc_poll_linkstat ? "" : ", and link status");
+
+ if (sc->sc_msix)
+ error = aq_setup_msix(sc, pa, sc->sc_nqueues,
+ sc->sc_use_txrx_independent_intr, !sc->sc_poll_linkstat);
+ else
+ error = ENODEV;
+
+ if (error != 0) {
+ /* if MSI-X failed, fallback to MSI with single queue */
+ sc->sc_use_txrx_independent_intr = false;
+ sc->sc_poll_linkstat = false;
+ sc->sc_msix = false;
+ sc->sc_nqueues = 1;
+ error = aq_setup_legacy(sc, pa, PCI_INTR_TYPE_MSI);
+ }
+ if (error != 0) {
+ /* if MSI failed, fallback to INTx */
+ error = aq_setup_legacy(sc, pa, PCI_INTR_TYPE_INTX);
+ }
+ if (error != 0)
+ return;
+
+ callout_init(&sc->sc_tick_ch, 0);
+ callout_setfunc(&sc->sc_tick_ch, aq_tick, sc);
+
+ sc->sc_intr_moderation_enable = CONFIG_INTR_MODERATION_ENABLE;
+
+ if (sc->sc_msix && (sc->sc_nqueues > 1))
+ sc->sc_rss_enable = true;
+ else
+ sc->sc_rss_enable = false;
+
+ error = aq_txrx_rings_alloc(sc);
+ if (error != 0)
+ goto attach_failure;
+
+ error = aq_fw_reset(sc);
+ if (error != 0)
+ goto attach_failure;
+
+ error = aq_fw_version_init(sc);
+ if (error != 0)
+ goto attach_failure;
+
+ error = aq_hw_init_ucp(sc);
+ if (error < 0)
+ goto attach_failure;
+
+ KASSERT(sc->sc_mbox_addr != 0);
+ error = aq_hw_reset(sc);
+ if (error != 0)
+ goto attach_failure;
+
+ aq_get_mac_addr(sc);
+ aq_init_rss(sc);
+
+ error = aq_hw_init(sc); /* initialize and interrupts */
+ if (error != 0)
+ goto attach_failure;
+
+ sc->sc_media_type = aqp->aq_media_type;
+ sc->sc_available_rates = aqp->aq_available_rates;
+
+ sc->sc_ethercom.ec_ifmedia = &sc->sc_media;
+ ifmedia_init(&sc->sc_media, IFM_IMASK,
+ aq_ifmedia_change, aq_ifmedia_status);
+ aq_initmedia(sc);
+
+ strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ);
+ ifp->if_softc = sc;
+ ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
+ ifp->if_baudrate = IF_Gbps(10);
+ ifp->if_init = aq_init;
+ ifp->if_ioctl = aq_ioctl;
+ if (sc->sc_msix && (sc->sc_nqueues > 1))
+ ifp->if_transmit = aq_transmit;
+ ifp->if_start = aq_start;
+ ifp->if_stop = aq_stop;
+ ifp->if_watchdog = aq_watchdog;
+ IFQ_SET_READY(&ifp->if_snd);
+
+ /* initialize capabilities */
+ sc->sc_ethercom.ec_capabilities = 0;
+ sc->sc_ethercom.ec_capenable = 0;
+#if notyet
+ /* TODO */
+ sc->sc_ethercom.ec_capabilities |= ETHERCAP_EEE;
+ sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_HWFILTER;
+#endif
+ sc->sc_ethercom.ec_capabilities |=
+ ETHERCAP_JUMBO_MTU |
+ ETHERCAP_VLAN_MTU |
+ ETHERCAP_VLAN_HWTAGGING;
+ sc->sc_ethercom.ec_capenable |=
+ ETHERCAP_VLAN_HWTAGGING;
+
+ ifp->if_capabilities = 0;
+ ifp->if_capenable = 0;
+#ifdef CONFIG_LRO_SUPPORT
+ ifp->if_capabilities |= IFCAP_LRO;
+ ifp->if_capenable |= IFCAP_LRO;
+#endif
+#if notyet
+ /* TSO */
+ ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6;
+#endif
+
+#if notyet
+ /*
+ * XXX:
+ * Rx L4 CSUM doesn't work well for fragment packet.
+ * aq marks 'CHEDKED' and 'BAD' for them.
+ * we need to ignore (clear) hw-csum flags if the packet is fragmented
+ *
+ * TODO: test with LRO enabled
+ */
+ ifp->if_capabilities |= IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_TCPv6_Rx;
+ ifp->if_capabilities |= IFCAP_CSUM_UDPv4_Rx | IFCAP_CSUM_UDPv6_Rx;
+#endif
+ /* TX hardware checksum offloadding */
+ ifp->if_capabilities |= IFCAP_CSUM_IPv4_Tx;
+ ifp->if_capabilities |= IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv6_Tx;
+ ifp->if_capabilities |= IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv6_Tx;
+ /* RX hardware checksum offloadding */
+ ifp->if_capabilities |= IFCAP_CSUM_IPv4_Rx;
+
+ if_attach(ifp);
+ if_deferred_start_init(ifp, NULL);
+ ether_ifattach(ifp, sc->sc_enaddr.ether_addr_octet);
+ ether_set_ifflags_cb(&sc->sc_ethercom, aq_ifflags_cb);
+
+ aq_enable_intr(sc, true, false); /* only intr about link */
+
+ /* update media */
+ aq_ifmedia_change(ifp);
+
+#ifdef AQ_EVENT_COUNTERS
+ /* get starting statistics values */
+ if (sc->sc_fw_ops != NULL && sc->sc_fw_ops->get_stats != NULL &&
+ sc->sc_fw_ops->get_stats(sc, &sc->sc_statistics[0]) == 0) {
+ sc->sc_poll_statistics = true;
+ }
+
+ AQ_EVCNT_ATTACH_MISC(sc, uprc, "RX unicast packet");
+ AQ_EVCNT_ATTACH_MISC(sc, bprc, "RX broadcast packet");
+ AQ_EVCNT_ATTACH_MISC(sc, mprc, "RX multicast packet");
+ AQ_EVCNT_ATTACH_MISC(sc, erpr, "RX error packet");
+ AQ_EVCNT_ATTACH_MISC(sc, ubrc, "RX unicast bytes");
+ AQ_EVCNT_ATTACH_MISC(sc, bbrc, "RX broadcast bytes");
+ AQ_EVCNT_ATTACH_MISC(sc, mbrc, "RX multicast bytes");
+ AQ_EVCNT_ATTACH_MISC(sc, prc, "RX good packet");
+ AQ_EVCNT_ATTACH_MISC(sc, uptc, "TX unicast packet");
+ AQ_EVCNT_ATTACH_MISC(sc, bptc, "TX broadcast packet");
+ AQ_EVCNT_ATTACH_MISC(sc, mptc, "TX multicast packet");
+ AQ_EVCNT_ATTACH_MISC(sc, erpt, "TX error packet");
+ AQ_EVCNT_ATTACH_MISC(sc, ubtc, "TX unicast bytes");
+ AQ_EVCNT_ATTACH_MISC(sc, bbtc, "TX broadcast bytes");
+ AQ_EVCNT_ATTACH_MISC(sc, mbtc, "TX multicast bytes");
+ AQ_EVCNT_ATTACH_MISC(sc, ptc, "TX good packet");
+ AQ_EVCNT_ATTACH_MISC(sc, dpc, "DMA drop packet");
+ AQ_EVCNT_ATTACH_MISC(sc, cprc, "RX coalesced packet");
+#endif
+
+ return;
+
+ attach_failure:
+ aq_detach(self, 0);
+}
+
+static int
+aq_detach(device_t self, int flags __unused)
+{
+ struct aq_softc *sc = device_private(self);
+ struct ifnet *ifp = &sc->sc_ethercom.ec_if;
+ int i, s;
+
+ if (sc->sc_iosize != 0) {
+ if (ifp->if_softc != NULL) {
+ s = splnet();
+ aq_stop(ifp, 0);
+ splx(s);
+ }
+
+ for (i = 0; i < AQ_NINTR_MAX; i++) {
+ if (sc->sc_ihs[i] != NULL) {
+ pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[i]);
+ sc->sc_ihs[i] = NULL;
+ }
+ }
+ if (sc->sc_nintrs > 0) {
+ pci_intr_release(sc->sc_pc, sc->sc_intrs,
+ sc->sc_nintrs);
+ sc->sc_intrs = NULL;
+ sc->sc_nintrs = 0;
+ }
+
+ aq_txrx_rings_free(sc);
+
+ if (ifp->if_softc != NULL) {
+ ether_ifdetach(ifp);
+ if_detach(ifp);
+ }
+
+ aprint_debug_dev(sc->sc_dev, "%s: bus_space_unmap\n", __func__);
+ bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_iosize);
+ sc->sc_iosize = 0;
+ }
+
+ callout_stop(&sc->sc_tick_ch);
+
+#ifdef AQ_EVENT_COUNTERS
+ AQ_EVCNT_DETACH(sc, uprc);
+ AQ_EVCNT_DETACH(sc, mprc);
+ AQ_EVCNT_DETACH(sc, bprc);
+ AQ_EVCNT_DETACH(sc, erpt);
+ AQ_EVCNT_DETACH(sc, uptc);
+ AQ_EVCNT_DETACH(sc, mptc);
+ AQ_EVCNT_DETACH(sc, bptc);
+ AQ_EVCNT_DETACH(sc, erpr);
+ AQ_EVCNT_DETACH(sc, mbtc);
+ AQ_EVCNT_DETACH(sc, bbtc);
+ AQ_EVCNT_DETACH(sc, mbrc);
+ AQ_EVCNT_DETACH(sc, bbrc);
+ AQ_EVCNT_DETACH(sc, ubrc);
+ AQ_EVCNT_DETACH(sc, ubtc);
+ AQ_EVCNT_DETACH(sc, ptc);
+ AQ_EVCNT_DETACH(sc, prc);
+ AQ_EVCNT_DETACH(sc, dpc);
+ AQ_EVCNT_DETACH(sc, cprc);
+#endif
+
+ mutex_destroy(&sc->sc_mutex);
+
+ return 0;
+}
+
+static int
+aq_establish_intr(struct aq_softc *sc, int intno, kcpuset_t *affinity,
+ int (*func)(void *), void *arg, const char *xname)
+{
+ char intrbuf[PCI_INTRSTR_LEN];
+ pci_chipset_tag_t pc = sc->sc_pc;
+ void *vih;
+ const char *intrstr = NULL;
+
+ intrstr = pci_intr_string(pc, sc->sc_intrs[intno], intrbuf,
+ sizeof(intrbuf));
+
+ pci_intr_setattr(pc, &sc->sc_intrs[intno], PCI_INTR_MPSAFE, true);
+
+ vih = pci_intr_establish_xname(pc, sc->sc_intrs[intno],
+ IPL_NET, func, arg, xname);
+ if (vih == NULL) {
+ aprint_error_dev(sc->sc_dev,
+ "unable to establish MSI-X%s%s for %s\n",
+ intrstr ? " at " : "",
+ intrstr ? intrstr : "", xname);
+ return EIO;
+ }
+ sc->sc_ihs[intno] = vih;
+
+ if (affinity != NULL) {
+ /* Round-robin affinity */
+ kcpuset_zero(affinity);
+ kcpuset_set(affinity, intno % ncpu);
+ interrupt_distribute(vih, affinity, NULL);
+ }
+
+ return 0;
+}
+
+static int
+aq_establish_msix_intr(struct aq_softc *sc, bool txrx_independent,
+ bool linkintr)
+{
+ kcpuset_t *affinity;
+ int error, intno, i;
+ char intr_xname[INTRDEVNAMEBUF];
+
+ kcpuset_create(&affinity, false);
+
+ intno = 0;
+
+ if (txrx_independent) {
+ for (i = 0; i < sc->sc_nqueues; i++) {
+ snprintf(intr_xname, sizeof(intr_xname), "%s RX%d",
+ device_xname(sc->sc_dev), i);
+ sc->sc_rx_irq[i] = intno;
+ error = aq_establish_intr(sc, intno++, affinity,
+ aq_rx_intr, &sc->sc_queue[i].rxring, intr_xname);
+ if (error != 0)
+ goto fail;
+ }
+ for (i = 0; i < sc->sc_nqueues; i++) {
+ snprintf(intr_xname, sizeof(intr_xname), "%s TX%d",
+ device_xname(sc->sc_dev), i);
+ sc->sc_tx_irq[i] = intno;
+ error = aq_establish_intr(sc, intno++, affinity,
+ aq_tx_intr, &sc->sc_queue[i].txring, intr_xname);
+ if (error != 0)
+ goto fail;
+ }
+ } else {
+ for (i = 0; i < sc->sc_nqueues; i++) {
+ snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d",
+ device_xname(sc->sc_dev), i);
+ sc->sc_rx_irq[i] = intno;
+ sc->sc_tx_irq[i] = intno;
+ error = aq_establish_intr(sc, intno++, affinity,
+ aq_txrx_intr, &sc->sc_queue[i], intr_xname);
+ if (error != 0)
+ goto fail;
+ }
+ }
+
+ if (linkintr) {
+ snprintf(intr_xname, sizeof(intr_xname), "%s LINK",
+ device_xname(sc->sc_dev));
+ sc->sc_linkstat_irq = intno;
+ error = aq_establish_intr(sc, intno++, affinity,
+ aq_link_intr, sc, intr_xname);
+ if (error != 0)
+ goto fail;
+ }
+
+ kcpuset_destroy(affinity);
+ return 0;
+
+ fail:
+ for (i = 0; i < AQ_NINTR_MAX; i++) {
+ if (sc->sc_ihs[i] != NULL) {
+ pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[i]);
+ sc->sc_ihs[i] = NULL;
+ }
+ }
+
+ kcpuset_destroy(affinity);
+ return ENOMEM;
+}
+
+static int
+aq_setup_msix(struct aq_softc *sc, struct pci_attach_args *pa, int nqueue,
+ bool txrx_independent, bool linkintr)
+{
+ int error, nintr;
+
+ if (txrx_independent)
+ nintr = nqueue * 2;
+ else
+ nintr = nqueue;
+
+ if (linkintr)
+ nintr++;
+
+ error = pci_msix_alloc_exact(pa, &sc->sc_intrs, nintr);
+ if (error != 0) {
+ aprint_error_dev(sc->sc_dev,
+ "failed to allocate MSI-X interrupts\n");
+ goto fail;
+ }
+
+ error = aq_establish_msix_intr(sc, txrx_independent, linkintr);
+ if (error == 0) {
+ sc->sc_nintrs = nintr;
+ } else {
+ pci_intr_release(sc->sc_pc, sc->sc_intrs, nintr);
+ sc->sc_nintrs = 0;
+ }
+ fail:
+ return error;
+
+}
+
+static int
+aq_setup_legacy(struct aq_softc *sc, struct pci_attach_args *pa,
+ pci_intr_type_t inttype)
+{
+ int counts[PCI_INTR_TYPE_SIZE];
+ int error, nintr;
+
+ nintr = 1;
+
+ memset(counts, 0, sizeof(counts));
+ counts[inttype] = nintr;
+
+ error = pci_intr_alloc(pa, &sc->sc_intrs, counts, inttype);
+ if (error != 0) {
+ aprint_error_dev(sc->sc_dev,
+ "failed to allocate%s interrupts\n",
+ (inttype == PCI_INTR_TYPE_MSI) ? " MSI" : "");
+ return error;
+ }
+ error = aq_establish_intr(sc, 0, NULL, aq_legacy_intr, sc,
+ device_xname(sc->sc_dev));
+ if (error == 0) {
+ sc->sc_nintrs = nintr;
+ } else {
+ pci_intr_release(sc->sc_pc, sc->sc_intrs, nintr);
+ sc->sc_nintrs = 0;
+ }
+ return error;
+}
+
+static void
+global_software_reset(struct aq_softc *sc)
+{
+ uint32_t v;
+
+ AQ_WRITE_REG_BIT(sc, RX_SYSCONTROL_REG, RX_SYSCONTROL_RESET_DIS, 0);
+ AQ_WRITE_REG_BIT(sc, TX_SYSCONTROL_REG, TX_SYSCONTROL_RESET_DIS, 0);
+ AQ_WRITE_REG_BIT(sc, FW_MPI_RESETCTRL_REG,
+ FW_MPI_RESETCTRL_RESET_DIS, 0);
+
+ v = AQ_READ_REG(sc, AQ_FW_SOFTRESET_REG);
+ v &= ~AQ_FW_SOFTRESET_DIS;
+ v |= AQ_FW_SOFTRESET_RESET;
+ AQ_WRITE_REG(sc, AQ_FW_SOFTRESET_REG, v);
+}
+
+static int
+mac_soft_reset_rbl(struct aq_softc *sc, aq_fw_bootloader_mode_t *mode)
+{
+ int timo;
+
+ aprint_debug_dev(sc->sc_dev, "RBL> MAC reset STARTED!\n");
+
+ AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x40e1);
+ AQ_WRITE_REG(sc, AQ_FW_GLB_CPU_SEM_REG(0), 1);
+ AQ_WRITE_REG(sc, AQ_MBOXIF_POWER_GATING_CONTROL_REG, 0);
+
+ /* MAC FW will reload PHY FW if 1E.1000.3 was cleaned - #undone */
+ AQ_WRITE_REG(sc, FW_BOOT_EXIT_CODE_REG, RBL_STATUS_DEAD);
+
+ global_software_reset(sc);
+
+ AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x40e0);
+
+ /* Wait for RBL to finish boot process. */
+#define RBL_TIMEOUT_MS 10000
+ uint16_t rbl_status;
+ for (timo = RBL_TIMEOUT_MS; timo > 0; timo--) {
+ rbl_status = AQ_READ_REG(sc, FW_BOOT_EXIT_CODE_REG) & 0xffff;
+ if (rbl_status != 0 && rbl_status != RBL_STATUS_DEAD)
+ break;
+ msec_delay(1);
+ }
+ if (timo <= 0) {
+ aprint_error_dev(sc->sc_dev,
+ "RBL> RBL restart failed: timeout\n");
+ return EBUSY;
+ }
+ switch (rbl_status) {
+ case RBL_STATUS_SUCCESS:
+ if (mode != NULL)
+ *mode = FW_BOOT_MODE_RBL_FLASH;
+ aprint_debug_dev(sc->sc_dev, "RBL> reset complete! [Flash]\n");
+ break;
+ case RBL_STATUS_HOST_BOOT:
+ if (mode != NULL)
+ *mode = FW_BOOT_MODE_RBL_HOST_BOOTLOAD;
+ aprint_debug_dev(sc->sc_dev,
+ "RBL> reset complete! [Host Bootload]\n");
+ break;
+ case RBL_STATUS_FAILURE:
+ default:
+ aprint_error_dev(sc->sc_dev,
+ "unknown RBL status 0x%x\n", rbl_status);
+ return EBUSY;
+ }
+
+ return 0;
+}
+
+static int
+mac_soft_reset_flb(struct aq_softc *sc)
+{
+ uint32_t v;
+ int timo;
+
+ AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x40e1);
+ /*
+ * Let Felicity hardware to complete SMBUS transaction before
+ * Global software reset.
+ */
+ msec_delay(50);
+
+ /*
+ * If SPI burst transaction was interrupted(before running the script),
+ * global software reset may not clear SPI interface.
+ * Clean it up manually before global reset.
+ */
+ AQ_WRITE_REG(sc, AQ_GLB_NVR_PROVISIONING2_REG, 0x00a0);
+ AQ_WRITE_REG(sc, AQ_GLB_NVR_INTERFACE1_REG, 0x009f);
+ AQ_WRITE_REG(sc, AQ_GLB_NVR_INTERFACE1_REG, 0x809f);
+ msec_delay(50);
+
+ v = AQ_READ_REG(sc, AQ_FW_SOFTRESET_REG);
+ v &= ~AQ_FW_SOFTRESET_DIS;
+ v |= AQ_FW_SOFTRESET_RESET;
+ AQ_WRITE_REG(sc, AQ_FW_SOFTRESET_REG, v);
+
+ /* Kickstart. */
+ AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x80e0);
+ AQ_WRITE_REG(sc, AQ_MBOXIF_POWER_GATING_CONTROL_REG, 0);
+ if (!sc->sc_fast_start_enabled)
+ AQ_WRITE_REG(sc, AQ_GLB_GENERAL_PROVISIONING9_REG, 1);
+
+ /*
+ * For the case SPI burst transaction was interrupted (by MCP reset
+ * above), wait until it is completed by hardware.
+ */
+ msec_delay(50);
+
+ /* MAC Kickstart */
+ if (!sc->sc_fast_start_enabled) {
+ AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x180e0);
+
+ uint32_t flb_status;
+ for (timo = 0; timo < 1000; timo++) {
+ flb_status = AQ_READ_REG(sc,
+ FW_MPI_DAISY_CHAIN_STATUS_REG) & 0x10;
+ if (flb_status != 0)
+ break;
+ msec_delay(1);
+ }
+ if (flb_status == 0) {
+ aprint_error_dev(sc->sc_dev,
+ "FLB> MAC kickstart failed: timed out\n");
+ return ETIMEDOUT;
+ }
+ aprint_debug_dev(sc->sc_dev,
+ "FLB> MAC kickstart done, %d ms\n", timo);
+ /* FW reset */
+ AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x80e0);
+ /*
+ * Let Felicity hardware complete SMBUS transaction before
+ * Global software reset.
+ */
+ msec_delay(50);
+ sc->sc_fast_start_enabled = true;
+ }
+ AQ_WRITE_REG(sc, AQ_FW_GLB_CPU_SEM_REG(0), 1);
+
+ /* PHY Kickstart: #undone */
+ global_software_reset(sc);
+
+ for (timo = 0; timo < 1000; timo++) {
+ if (AQ_READ_REG(sc, AQ_FW_VERSION_REG) != 0)
+ break;
+ msec_delay(10);
+ }
+ if (timo >= 1000) {
+ aprint_error_dev(sc->sc_dev, "FLB> Global Soft Reset failed\n");
+ return ETIMEDOUT;
+ }
+ aprint_debug_dev(sc->sc_dev, "FLB> F/W restart: %d ms\n", timo * 10);
+ return 0;
+
+}
+
+static int
+mac_soft_reset(struct aq_softc *sc, aq_fw_bootloader_mode_t *mode)
+{
+ if (sc->sc_rbl_enabled)
+ return mac_soft_reset_rbl(sc, mode);
+
+ if (mode != NULL)
+ *mode = FW_BOOT_MODE_FLB;
+ return mac_soft_reset_flb(sc);
+}
+
+static int
+aq_fw_read_version(struct aq_softc *sc)
+{
+ int i, error = EBUSY;
+#define MAC_FW_START_TIMEOUT_MS 10000
+ for (i = 0; i < MAC_FW_START_TIMEOUT_MS; i++) {
+ sc->sc_fw_version = AQ_READ_REG(sc, AQ_FW_VERSION_REG);
+ if (sc->sc_fw_version != 0) {
+ error = 0;
+ break;
+ }
+ delay(1000);
+ }
+ return error;
+}
+
+static int
+aq_fw_reset(struct aq_softc *sc)
+{
+ uint32_t ver, v, bootExitCode;
+ int i, error;
+
+ ver = AQ_READ_REG(sc, AQ_FW_VERSION_REG);
+
+ for (i = 1000; i > 0; i--) {
+ v = AQ_READ_REG(sc, FW_MPI_DAISY_CHAIN_STATUS_REG);
+ bootExitCode = AQ_READ_REG(sc, FW_BOOT_EXIT_CODE_REG);
+ if (v != 0x06000000 || bootExitCode != 0)
+ break;
+ }
+ if (i <= 0) {
+ aprint_error_dev(sc->sc_dev,
+ "F/W reset failed. Neither RBL nor FLB started\n");
+ return ETIMEDOUT;
+ }
+ sc->sc_rbl_enabled = (bootExitCode != 0);
+
+ /*
+ * Having FW version 0 is an indicator that cold start
+ * is in progress. This means two things:
+ * 1) Driver have to wait for FW/HW to finish boot (500ms giveup)
+ * 2) Driver may skip reset sequence and save time.
+ */
+ if (sc->sc_fast_start_enabled && (ver != 0)) {
+ error = aq_fw_read_version(sc);
+ /* Skip reset as it just completed */
+ if (error == 0)
+ return 0;
+ }
+
+ aq_fw_bootloader_mode_t mode = FW_BOOT_MODE_UNKNOWN;
+ error = mac_soft_reset(sc, &mode);
+ if (error != 0) {
+ aprint_error_dev(sc->sc_dev, "MAC reset failed: %d\n", error);
+ return error;
+ }
+
+ switch (mode) {
+ case FW_BOOT_MODE_FLB:
+ aprint_debug_dev(sc->sc_dev,
+ "FLB> F/W successfully loaded from flash.\n");
+ sc->sc_flash_present = true;
+ return aq_fw_read_version(sc);
+ case FW_BOOT_MODE_RBL_FLASH:
+ aprint_debug_dev(sc->sc_dev,
+ "RBL> F/W loaded from flash. Host Bootload disabled.\n");
+ sc->sc_flash_present = true;
+ return aq_fw_read_version(sc);
+ case FW_BOOT_MODE_UNKNOWN:
+ aprint_error_dev(sc->sc_dev,
+ "F/W bootload error: unknown bootloader type\n");
+ return ENOTSUP;
+ case FW_BOOT_MODE_RBL_HOST_BOOTLOAD:
+ aprint_debug_dev(sc->sc_dev, "RBL> Host Bootload mode\n");
+ break;
+ }
+
+ /*
+ * XXX: TODO: add support Host Boot
+ */
+ aprint_error_dev(sc->sc_dev,
+ "RBL> F/W Host Bootload not implemented\n");
+ return ENOTSUP;
+}
+
+static int
+aq_hw_reset(struct aq_softc *sc)
+{
+ int error;
+
+ /* disable irq */
+ AQ_WRITE_REG_BIT(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_RESET_DIS, 0);
+
+ /* apply */
+ AQ_WRITE_REG_BIT(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_RESET_IRQ, 1);
+
+ /* wait ack 10 times by 1ms */
+ WAIT_FOR(
+ (AQ_READ_REG(sc, AQ_INTR_CTRL_REG) & AQ_INTR_CTRL_RESET_IRQ) == 0,
+ 1000, 10, &error);
+ if (error != 0) {
+ aprint_error_dev(sc->sc_dev,
+ "atlantic: IRQ reset failed: %d\n", error);
+ return error;
+ }
+
+ return sc->sc_fw_ops->reset(sc);
+}
+
+static int
+aq_hw_init_ucp(struct aq_softc *sc)
+{
+ int timo;
+
+ if (FW_VERSION_MAJOR(sc) == 1) {
+ if (AQ_READ_REG(sc, FW1X_MPI_INIT2_REG) == 0) {
+ uint32_t data;
+ cprng_fast(&data, sizeof(data));
+ data &= 0xfefefefe;
+ data |= 0x02020202;
+ AQ_WRITE_REG(sc, FW1X_MPI_INIT2_REG, data);
+ }
+ AQ_WRITE_REG(sc, FW1X_MPI_INIT1_REG, 0);
+ }
+
+ for (timo = 100; timo > 0; timo--) {
+ sc->sc_mbox_addr = AQ_READ_REG(sc, FW_MPI_MBOX_ADDR_REG);
+ if (sc->sc_mbox_addr != 0)
+ break;
+ delay(1000);
+ }
+
+#define AQ_FW_MIN_VERSION 0x01050006
+#define AQ_FW_MIN_VERSION_STR "1.5.6"
+ if (sc->sc_fw_version < AQ_FW_MIN_VERSION) {
+ aprint_error_dev(sc->sc_dev,
+ "atlantic: wrong FW version: " AQ_FW_MIN_VERSION_STR
+ " or later required, this is %d.%d.%d\n",
+ FW_VERSION_MAJOR(sc),
+ FW_VERSION_MINOR(sc),
+ FW_VERSION_BUILD(sc));
+ return ENOTSUP;
+ }
+
+ return 0;
+}
+
+static int
+aq_fw_version_init(struct aq_softc *sc)
+{
+ int error = 0;
+ char fw_vers[sizeof("F/W version xxxxx.xxxxx.xxxxx")];
+
+ if (FW_VERSION_MAJOR(sc) == 1) {
+ sc->sc_fw_ops = &aq_fw1x_ops;
+ } else if ((FW_VERSION_MAJOR(sc) == 2) || (FW_VERSION_MAJOR(sc) == 3)) {
+ sc->sc_fw_ops = &aq_fw2x_ops;
+ } else {
+ aprint_error_dev(sc->sc_dev,
+ "Unsupported F/W version %d.%d.%d\n",
+ FW_VERSION_MAJOR(sc), FW_VERSION_MINOR(sc),
+ FW_VERSION_BUILD(sc));
+ return ENOTSUP;
+ }
+ snprintf(fw_vers, sizeof(fw_vers), "F/W version %d.%d.%d",
+ FW_VERSION_MAJOR(sc), FW_VERSION_MINOR(sc), FW_VERSION_BUILD(sc));
+
+ /* detect revision */
+ uint32_t hwrev = AQ_READ_REG(sc, AQ_HW_REVISION_REG);
+ switch (hwrev & 0x0000000f) {
+ case 0x01:
+ aprint_normal_dev(sc->sc_dev, "Atlantic revision A0, %s\n",
+ fw_vers);
+ sc->sc_features |= FEATURES_REV_A0 |
+ FEATURES_MPI_AQ | FEATURES_MIPS;
+ break;
+ case 0x02:
+ aprint_normal_dev(sc->sc_dev, "Atlantic revision B0, %s\n",
+ fw_vers);
+ sc->sc_features |= FEATURES_REV_B0 |
+ FEATURES_MPI_AQ | FEATURES_MIPS |
+ FEATURES_TPO2 | FEATURES_RPF2;
+ break;
+ case 0x0A:
+ aprint_normal_dev(sc->sc_dev, "Atlantic revision B1, %s\n",
+ fw_vers);
+ sc->sc_features |= FEATURES_REV_B1 |
+ FEATURES_MPI_AQ | FEATURES_MIPS |
+ FEATURES_TPO2 | FEATURES_RPF2;
+ break;
+ default:
+ aprint_error_dev(sc->sc_dev,
+ "Unknown revision (0x%08x)\n", hwrev);
+ error = ENOTSUP;
+ break;
+ }
+ return error;
+}
+
+static int
+fw1x_reset(struct aq_softc *sc)
+{
+ struct aq_mailbox_header mbox;
+ const int retryCount = 1000;
+ uint32_t tid0;
+ int i;
+
+ tid0 = ~0; /*< Initial value of MBOX transactionId. */
+
+ for (i = 0; i < retryCount; ++i) {
+ /*
+ * Read the beginning of Statistics structure to capture
+ * the Transaction ID.
+ */
+ aq_fw_downld_dwords(sc, sc->sc_mbox_addr,
+ (uint32_t *)&mbox, sizeof(mbox) / sizeof(uint32_t));
+
+ /* Successfully read the stats. */
+ if (tid0 == ~0U) {
+ /* We have read the initial value. */
+ tid0 = mbox.transaction_id;
+ continue;
+ } else if (mbox.transaction_id != tid0) {
+ /*
+ * Compare transaction ID to initial value.
+ * If it's different means f/w is alive.
+ * We're done.
+ */
+ return 0;
+ }
+
+ /*
+ * Transaction ID value haven't changed since last time.
+ * Try reading the stats again.
+ */
+ delay(10);
+ }
+ aprint_error_dev(sc->sc_dev, "F/W 1.x reset finalize timeout\n");
+ return EBUSY;
+}
+
+static int
+fw1x_set_mode(struct aq_softc *sc, aq_hw_fw_mpi_state_t mode,
+ aq_link_speed_t speed, aq_link_fc_t fc, aq_link_eee_t eee)
+{
+ uint32_t mpictrl = 0;
+ uint32_t mpispeed = 0;
+
+ if (speed & AQ_LINK_10G)
+ mpispeed |= FW1X_CTRL_10G;
+ if (speed & AQ_LINK_5G)
+ mpispeed |= (FW1X_CTRL_5G | FW1X_CTRL_5GSR);
+ if (speed & AQ_LINK_2G5)
+ mpispeed |= FW1X_CTRL_2G5;
+ if (speed & AQ_LINK_1G)
+ mpispeed |= FW1X_CTRL_1G;
+ if (speed & AQ_LINK_100M)
+ mpispeed |= FW1X_CTRL_100M;
+
+ mpictrl |= __SHIFTIN(mode, FW1X_MPI_STATE_MODE);
+ mpictrl |= __SHIFTIN(mpispeed, FW1X_MPI_STATE_SPEED);
+ AQ_WRITE_REG(sc, FW1X_MPI_CONTROL_REG, mpictrl);
+ return 0;
+}
+
+static int
+fw1x_get_mode(struct aq_softc *sc, aq_hw_fw_mpi_state_t *modep,
+ aq_link_speed_t *speedp, aq_link_fc_t *fcp, aq_link_eee_t *eeep)
+{
+ uint32_t mpistate, mpi_speed;
+ aq_link_speed_t speed = AQ_LINK_NONE;
+
+ mpistate = AQ_READ_REG(sc, FW1X_MPI_STATE_REG);
+
+ if (modep != NULL)
+ *modep = __SHIFTOUT(mpistate, FW1X_MPI_STATE_MODE);
+
+ mpi_speed = __SHIFTOUT(mpistate, FW1X_MPI_STATE_SPEED);
+ if (mpi_speed & FW1X_CTRL_10G)
+ speed = AQ_LINK_10G;
+ else if (mpi_speed & (FW1X_CTRL_5G|FW1X_CTRL_5GSR))
+ speed = AQ_LINK_5G;
+ else if (mpi_speed & FW1X_CTRL_2G5)
+ speed = AQ_LINK_2G5;
+ else if (mpi_speed & FW1X_CTRL_1G)
+ speed = AQ_LINK_1G;
+ else if (mpi_speed & FW1X_CTRL_100M)
+ speed = AQ_LINK_100M;
+
+ if (speedp != NULL)
+ *speedp = speed;
+
+ if (fcp != NULL)
+ *fcp = AQ_FC_NONE;
+
+ if (eeep != NULL)
+ *eeep = AQ_EEE_DISABLE;
+
+ return 0;
+}
+
+static int
+fw1x_get_stats(struct aq_softc *sc, aq_hw_stats_s_t *stats)
+{
+ int error;
+
+ error = aq_fw_downld_dwords(sc,
+ sc->sc_mbox_addr + offsetof(fw1x_mailbox_t, msm), (uint32_t *)stats,
+ sizeof(aq_hw_stats_s_t) / sizeof(uint32_t));
+ if (error < 0) {
+ device_printf(sc->sc_dev,
+ "fw1x> download statistics data FAILED, error %d", error);
+ return error;
+ }
+
+ stats->dpc = AQ_READ_REG(sc, RX_DMA_DROP_PKT_CNT_REG);
+ stats->cprc = AQ_READ_REG(sc, RX_DMA_COALESCED_PKT_CNT_REG);
+ return 0;
+}
+
+static int
+fw2x_reset(struct aq_softc *sc)
+{
+ fw2x_capabilities_t caps = { 0 };
+ int error;
+
+ error = aq_fw_downld_dwords(sc,
+ sc->sc_mbox_addr + offsetof(fw2x_mailbox_t, caps),
+ (uint32_t *)&caps, sizeof caps / sizeof(uint32_t));
+ if (error != 0) {
+ aprint_error_dev(sc->sc_dev,
+ "fw2x> can't get F/W capabilities mask, error %d\n",
+ error);
+ return error;
+ }
+ sc->sc_fw_caps = caps.caps_lo | ((uint64_t)caps.caps_hi << 32);
+
+ char buf[256];
+ snprintb(buf, sizeof(buf), FW2X_SNPRINTB, sc->sc_fw_caps);
+ aprint_verbose_dev(sc->sc_dev, "fw2x> F/W capabilities=%s\n", buf);
+
+ return 0;
+}
+
+static int
+fw2x_set_mode(struct aq_softc *sc, aq_hw_fw_mpi_state_t mode,
+ aq_link_speed_t speed, aq_link_fc_t fc, aq_link_eee_t eee)
+{
+ uint64_t mpi_ctrl = AQ_READ64_REG(sc, FW2X_MPI_CONTROL_REG);
+
+ switch (mode) {
+ case MPI_INIT:
+ mpi_ctrl &= ~FW2X_CTRL_RATE_MASK;
+ if (speed & AQ_LINK_10G)
+ mpi_ctrl |= FW2X_CTRL_RATE_10G;
+ if (speed & AQ_LINK_5G)
+ mpi_ctrl |= FW2X_CTRL_RATE_5G;
+ if (speed & AQ_LINK_2G5)
+ mpi_ctrl |= FW2X_CTRL_RATE_2G5;
+ if (speed & AQ_LINK_1G)
+ mpi_ctrl |= FW2X_CTRL_RATE_1G;
+ if (speed & AQ_LINK_100M)
+ mpi_ctrl |= FW2X_CTRL_RATE_100M;
+
+ mpi_ctrl &= ~FW2X_CTRL_LINK_DROP;
+
+ mpi_ctrl &= ~FW2X_CTRL_EEE_MASK;
+ if (eee == AQ_EEE_ENABLE)
+ mpi_ctrl |= FW2X_CTRL_EEE_MASK;
+
+ mpi_ctrl &= ~(FW2X_CTRL_PAUSE | FW2X_CTRL_ASYMMETRIC_PAUSE);
+ if (fc & AQ_FC_RX)
+ mpi_ctrl |= FW2X_CTRL_PAUSE;
+ if (fc & AQ_FC_TX)
+ mpi_ctrl |= FW2X_CTRL_ASYMMETRIC_PAUSE;
+ break;
+ case MPI_DEINIT:
+ mpi_ctrl &= ~(FW2X_CTRL_RATE_MASK | FW2X_CTRL_EEE_MASK);
+ mpi_ctrl &= ~(FW2X_CTRL_PAUSE | FW2X_CTRL_ASYMMETRIC_PAUSE);
+ break;
+ default:
+ device_printf(sc->sc_dev, "fw2x> unknown MPI state %d\n", mode);
+ return EINVAL;
+ }
+
+ AQ_WRITE64_REG(sc, FW2X_MPI_CONTROL_REG, mpi_ctrl);
+ return 0;
+}
+
+static int
+fw2x_get_mode(struct aq_softc *sc, aq_hw_fw_mpi_state_t *modep,
+ aq_link_speed_t *speedp, aq_link_fc_t *fcp, aq_link_eee_t *eeep)
+{
+ uint64_t mpi_state = AQ_READ64_REG(sc, FW2X_MPI_STATE_REG);
+
+ if (modep != NULL) {
+ uint64_t mpi_ctrl = AQ_READ64_REG(sc, FW2X_MPI_CONTROL_REG);
+ if (mpi_ctrl & FW2X_CTRL_RATE_MASK)
+ *modep = MPI_INIT;
+ else
+ *modep = MPI_DEINIT;
+ }
+
+ aq_link_speed_t speed = AQ_LINK_NONE;
+ if (mpi_state & FW2X_CTRL_RATE_10G)
+ speed = AQ_LINK_10G;
+ else if (mpi_state & FW2X_CTRL_RATE_5G)
+ speed = AQ_LINK_5G;
+ else if (mpi_state & FW2X_CTRL_RATE_2G5)
+ speed = AQ_LINK_2G5;
+ else if (mpi_state & FW2X_CTRL_RATE_1G)
+ speed = AQ_LINK_1G;
+ else if (mpi_state & FW2X_CTRL_RATE_100M)
+ speed = AQ_LINK_100M;
+
+ if (speedp != NULL)
+ *speedp = speed;
+
+ aq_link_fc_t fc = AQ_FC_NONE;
+ if (mpi_state & FW2X_CTRL_PAUSE)
+ fc |= AQ_FC_RX;
+ if (mpi_state & FW2X_CTRL_ASYMMETRIC_PAUSE)
+ fc |= AQ_FC_TX;
+ if (fcp != NULL)
+ *fcp = fc;
+
+ /* XXX: TODO: EEE */
+ if (eeep != NULL)
+ *eeep = AQ_EEE_DISABLE;
+
+ return 0;
+}
+
+static int
+toggle_mpi_ctrl_and_wait(struct aq_softc *sc, uint64_t mask,
+ uint32_t timeout_ms, uint32_t try_count)
+{
+ uint64_t mpi_ctrl = AQ_READ64_REG(sc, FW2X_MPI_CONTROL_REG);
+ uint64_t mpi_state = AQ_READ64_REG(sc, FW2X_MPI_STATE_REG);
+ int error;
+
+ /* First, check that control and state values are consistent */
+ if ((mpi_ctrl & mask) != (mpi_state & mask)) {
+ device_printf(sc->sc_dev,
+ "fw2x> MPI control (%#llx) and state (%#llx)"
+ " are not consistent for mask %#llx!\n",
+ (unsigned long long)mpi_ctrl, (unsigned long long)mpi_state,
+ (unsigned long long)mask);
+ return EINVAL;
+ }
+
+ /* Invert bits (toggle) in control register */
+ mpi_ctrl ^= mask;
+ AQ_WRITE64_REG(sc, FW2X_MPI_CONTROL_REG, mpi_ctrl);
+
+ /* Clear all bits except masked */
+ mpi_ctrl &= mask;
+
+ /* Wait for FW reflecting change in state register */
+ WAIT_FOR((AQ_READ64_REG(sc, FW2X_MPI_CONTROL_REG) & mask) == mpi_ctrl,
+ 1000 * timeout_ms, try_count, &error);
+ if (error != 0) {
+ device_printf(sc->sc_dev,
+ "f/w2x> timeout while waiting for response"
+ " in state register for bit %#llx!",
+ (unsigned long long)mask);
+ return error;
+ }
+ return 0;
+}
+
+static int
+fw2x_get_stats(struct aq_softc *sc, aq_hw_stats_s_t *stats)
+{
+ int error;
+
+ /* Say to F/W to update the statistics */
+ error = toggle_mpi_ctrl_and_wait(sc, FW2X_CTRL_STATISTICS, 1, 25);
+ if (error != 0) {
+ device_printf(sc->sc_dev,
+ "fw2x> statistics update error %d\n", error);
+ return error;
+ }
+
+ CTASSERT(sizeof(fw2x_msm_statistics_t) <= sizeof(struct aq_hw_stats_s));
+ error = aq_fw_downld_dwords(sc,
+ sc->sc_mbox_addr + offsetof(fw2x_mailbox_t, msm), (uint32_t *)stats,
+ sizeof(fw2x_msm_statistics_t) / sizeof(uint32_t));
+ if (error != 0) {
+ device_printf(sc->sc_dev,
+ "fw2x> download statistics data FAILED, error %d", error);
+ return error;
+ }
+ stats->dpc = AQ_READ_REG(sc, RX_DMA_DROP_PKT_CNT_REG);
+ stats->cprc = AQ_READ_REG(sc, RX_DMA_COALESCED_PKT_CNT_REG);
+
+ return 0;
+}
+
+static int
+aq_fw_downld_dwords(struct aq_softc *sc, uint32_t addr, uint32_t *p,
+ uint32_t cnt)
+{
+ uint32_t v;
+ int error = 0;
+
+ WAIT_FOR(AQ_READ_REG(sc, AQ_FW_SEM_RAM_REG) == 1, 1, 10000, &error);
+ if (error != 0) {
+ AQ_WRITE_REG(sc, AQ_FW_SEM_RAM_REG, 1);
+ v = AQ_READ_REG(sc, AQ_FW_SEM_RAM_REG);
+ if (v == 0) {
+ device_printf(sc->sc_dev,
+ "%s:%d: timeout\n", __func__, __LINE__);
+ return ETIMEDOUT;
+ }
+ }
+
+ AQ_WRITE_REG(sc, AQ_FW_MBOX_ADDR_REG, addr);
+
+ error = 0;
+ for (; cnt > 0 && error == 0; cnt--) {
+ /* execute mailbox interface */
+ AQ_WRITE_REG_BIT(sc, AQ_FW_MBOX_CMD_REG,
+ AQ_FW_MBOX_CMD_EXECUTE, 1);
+ if (sc->sc_features & FEATURES_REV_B1) {
+ WAIT_FOR(AQ_READ_REG(sc, AQ_FW_MBOX_ADDR_REG) != addr,
+ 1, 1000, &error);
+ } else {
+ WAIT_FOR((AQ_READ_REG(sc, AQ_FW_MBOX_CMD_REG) &
+ AQ_FW_MBOX_CMD_BUSY) == 0,
+ 1, 1000, &error);
+ }
+ *p++ = AQ_READ_REG(sc, AQ_FW_MBOX_VAL_REG);
+ addr += sizeof(uint32_t);
+ }
+ AQ_WRITE_REG(sc, AQ_FW_SEM_RAM_REG, 1);
+
+ if (error != 0)
+ device_printf(sc->sc_dev,
+ "%s:%d: timeout\n", __func__, __LINE__);
+
+ return error;
+}
+
+/* read my mac address */
+static int
+aq_get_mac_addr(struct aq_softc *sc)
+{
+ uint32_t mac_addr[2];
+ uint32_t efuse_shadow_addr;
+ int err;
+
+ efuse_shadow_addr = 0;
+ if (FW_VERSION_MAJOR(sc) >= 2)
+ efuse_shadow_addr = AQ_READ_REG(sc, FW2X_MPI_EFUSEADDR_REG);
+ else
+ efuse_shadow_addr = AQ_READ_REG(sc, FW1X_MPI_EFUSEADDR_REG);
+
+ if (efuse_shadow_addr == 0) {
+ aprint_error_dev(sc->sc_dev, "cannot get efuse addr\n");
+ return ENXIO;
+ }
+
+ memset(mac_addr, 0, sizeof(mac_addr));
+ err = aq_fw_downld_dwords(sc, efuse_shadow_addr + (40 * 4),
+ mac_addr, __arraycount(mac_addr));
+ if (err < 0)
+ return err;
+
+ if (mac_addr[0] == 0 && mac_addr[1] == 0) {
+ aprint_error_dev(sc->sc_dev, "mac address not found\n");
+ return ENXIO;
+ }
+
+ mac_addr[0] = bswap32(mac_addr[0]);
+ mac_addr[1] = bswap32(mac_addr[1]);
+
+ memcpy(sc->sc_enaddr.ether_addr_octet,
+ (uint8_t *)mac_addr, ETHER_ADDR_LEN);
+ aprint_normal_dev(sc->sc_dev, "Etheraddr: %s\n",
+ ether_sprintf(sc->sc_enaddr.ether_addr_octet));
+
+ return 0;
+}
+
+/* set multicast filter. index 0 for own address */
+static int
+aq_set_mac_addr(struct aq_softc *sc, int index, uint8_t *enaddr)
+{
+ uint32_t h, l;
+
+ if (index >= AQ_HW_MAC_NUM)
+ return EINVAL;
+
+ if (enaddr == NULL) {
+ /* disable */
+ AQ_WRITE_REG_BIT(sc,
+ RPF_L2UC_MSW_REG(index), RPF_L2UC_MSW_EN, 0);
+ return 0;
+ }
+
+ h = (enaddr[0] << 8) | (enaddr[1]);
+ l = (enaddr[2] << 24) | (enaddr[3] << 16) |
+ (enaddr[4] << 8) | (enaddr[5]);
+
+ /* disable, set, and enable */
+ AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(index), RPF_L2UC_MSW_EN, 0);
+ AQ_WRITE_REG(sc, RPF_L2UC_LSW_REG(index), l);
+ AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(index),
+ RPF_L2UC_MSW_MACADDR_HI, h);
+ AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(index), RPF_L2UC_MSW_ACTION, 1);
+ AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(index), RPF_L2UC_MSW_EN, 1);
+
+ return 0;
+}
+
+static int
+aq_set_capability(struct aq_softc *sc)
+{
+ struct ifnet *ifp = &sc->sc_ethercom.ec_if;
+ int ip4csum_tx =
+ ((ifp->if_capenable & IFCAP_CSUM_IPv4_Tx) == 0) ? 0 : 1;
+ int ip4csum_rx =
+ ((ifp->if_capenable & IFCAP_CSUM_IPv4_Rx) == 0) ? 0 : 1;
+ int l4csum_tx = ((ifp->if_capenable &
+ (IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_UDPv4_Tx |
+ IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_UDPv6_Tx)) == 0) ? 0 : 1;
+ int l4csum_rx =
+ ((ifp->if_capenable & (IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx |
+ IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx)) == 0) ? 0 : 1;
+ uint32_t lso =
+ ((ifp->if_capenable & (IFCAP_TSOv4 | IFCAP_TSOv6)) == 0) ?
+ 0 : 0xffffffff;
+ uint32_t lro = ((ifp->if_capenable & IFCAP_LRO) == 0) ?
+ 0 : 0xffffffff;
+ uint32_t i, v;
+
+ /* TX checksums offloads*/
+ AQ_WRITE_REG_BIT(sc, TPO_HWCSUM_REG, TPO_HWCSUM_IP4CSUM_EN, ip4csum_tx);
+ AQ_WRITE_REG_BIT(sc, TPO_HWCSUM_REG, TPO_HWCSUM_L4CSUM_EN, l4csum_tx);
+
+ /* RX checksums offloads*/
+ AQ_WRITE_REG_BIT(sc, RPO_HWCSUM_REG, RPO_HWCSUM_IP4CSUM_EN, ip4csum_rx);
+ AQ_WRITE_REG_BIT(sc, RPO_HWCSUM_REG, RPO_HWCSUM_L4CSUM_EN, l4csum_rx);
+
+ /* LSO offloads*/
+ AQ_WRITE_REG(sc, TDM_LSO_EN_REG, lso);
+
+#define AQ_B0_LRO_RXD_MAX 16
+ v = (8 < AQ_B0_LRO_RXD_MAX) ? 3 :
+ (4 < AQ_B0_LRO_RXD_MAX) ? 2 :
+ (2 < AQ_B0_LRO_RXD_MAX) ? 1 : 0;
+ for (i = 0; i < AQ_RINGS_NUM; i++) {
+ AQ_WRITE_REG_BIT(sc, RPO_LRO_LDES_MAX_REG(i),
+ RPO_LRO_LDES_MAX_MASK(i), v);
+ }
+
+ AQ_WRITE_REG_BIT(sc, RPO_LRO_TB_DIV_REG, RPO_LRO_TB_DIV, 0x61a);
+ AQ_WRITE_REG_BIT(sc, RPO_LRO_INACTIVE_IVAL_REG,
+ RPO_LRO_INACTIVE_IVAL, 0);
+ /*
+ * the LRO timebase divider is 5 uS (0x61a),
+ * to get a maximum coalescing interval of 250 uS,
+ * we need to multiply by 50(0x32) to get
+ * the default value 250 uS
+ */
+ AQ_WRITE_REG_BIT(sc, RPO_LRO_MAX_COALESCING_IVAL_REG,
+ RPO_LRO_MAX_COALESCING_IVAL, 50);
+ AQ_WRITE_REG_BIT(sc, RPO_LRO_CONF_REG,
+ RPO_LRO_CONF_QSESSION_LIMIT, 1);
+ AQ_WRITE_REG_BIT(sc, RPO_LRO_CONF_REG,
+ RPO_LRO_CONF_TOTAL_DESC_LIMIT, 2);
+ AQ_WRITE_REG_BIT(sc, RPO_LRO_CONF_REG,
+ RPO_LRO_CONF_PATCHOPTIMIZATION_EN, 0);
+ AQ_WRITE_REG_BIT(sc, RPO_LRO_CONF_REG,
+ RPO_LRO_CONF_MIN_PAYLOAD_OF_FIRST_PKT, 10);
+ AQ_WRITE_REG(sc, RPO_LRO_RSC_MAX_REG, 1);
+ AQ_WRITE_REG(sc, RPO_LRO_ENABLE_REG, lro);
+
+ return 0;
+}
+
+static int
+aq_set_filter(struct aq_softc *sc)
+{
+ struct ifnet *ifp = &sc->sc_ethercom.ec_if;
+ struct ethercom *ec = &sc->sc_ethercom;
+ struct ether_multi *enm;
+ struct ether_multistep step;
+ int idx, error = 0;
+
+ if (ifp->if_flags & IFF_PROMISC) {
+ AQ_WRITE_REG_BIT(sc, RPF_L2BC_REG, RPF_L2BC_PROMISC,
+ (ifp->if_flags & IFF_PROMISC) ? 1 : 0);
+ ec->ec_flags |= ETHER_F_ALLMULTI;
+ goto done;
+ }
+
+ /* clear all table */
+ for (idx = 0; idx < AQ_HW_MAC_NUM; idx++) {
+ if (idx == AQ_HW_MAC_OWN) /* already used for own */
+ continue;
+ aq_set_mac_addr(sc, idx, NULL);
+ }
+
+ /* don't accept all multicast */
+ AQ_WRITE_REG_BIT(sc, RPF_MCAST_FILTER_MASK_REG,
+ RPF_MCAST_FILTER_MASK_ALLMULTI, 0);
+ AQ_WRITE_REG_BIT(sc, RPF_MCAST_FILTER_REG(0),
+ RPF_MCAST_FILTER_EN, 0);
+
+ idx = 0;
+ ETHER_LOCK(ec);
+ ETHER_FIRST_MULTI(step, ec, enm);
+ while (enm != NULL) {
+ if (idx == AQ_HW_MAC_OWN)
+ idx++;
+
+ if ((idx >= AQ_HW_MAC_NUM) ||
+ memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) {
+ /*
+ * too many filters.
+ * fallback to accept all multicast addresses.
+ */
+ AQ_WRITE_REG_BIT(sc, RPF_MCAST_FILTER_MASK_REG,
+ RPF_MCAST_FILTER_MASK_ALLMULTI, 1);
+ AQ_WRITE_REG_BIT(sc, RPF_MCAST_FILTER_REG(0),
+ RPF_MCAST_FILTER_EN, 1);
+ ec->ec_flags |= ETHER_F_ALLMULTI;
+ ETHER_UNLOCK(ec);
+ goto done;
+ }
+
+ /* add a filter */
+ aq_set_mac_addr(sc, idx++, enm->enm_addrlo);
+
+ ETHER_NEXT_MULTI(step, enm);
+ }
+ ec->ec_flags &= ~ETHER_F_ALLMULTI;
+ ETHER_UNLOCK(ec);
+
+ done:
+ return error;
+}
+
+static void
+aq_mediastatus_update(struct aq_softc *sc)
+{
+ sc->sc_media_active = 0;
+
+ if (sc->sc_link_fc & AQ_FC_RX)
+ sc->sc_media_active |= IFM_ETH_RXPAUSE;
+ if (sc->sc_link_fc & AQ_FC_TX)
+ sc->sc_media_active |= IFM_ETH_TXPAUSE;
+
+ switch (sc->sc_link_rate) {
+ case AQ_LINK_100M:
+ /* XXX: need to detect fulldup or halfdup */
+ sc->sc_media_active |= IFM_100_TX | IFM_FDX;
+ break;
+ case AQ_LINK_1G:
+ sc->sc_media_active |= IFM_1000_T | IFM_FDX;
+ break;
+ case AQ_LINK_2G5:
+ sc->sc_media_active |= IFM_2500_T | IFM_FDX;
+ break;
+ case AQ_LINK_5G:
+ sc->sc_media_active |= IFM_5000_T | IFM_FDX;
+ break;
+ case AQ_LINK_10G:
+ sc->sc_media_active |= IFM_10G_T | IFM_FDX;
+ break;
+ default:
+ sc->sc_media_active |= IFM_NONE;
+ break;
+ }
+}
+
+static int
+aq_ifmedia_change(struct ifnet * const ifp)
+{
+ struct aq_softc *sc = ifp->if_softc;
+ aq_link_speed_t rate = AQ_LINK_NONE;
+ aq_link_fc_t fc = AQ_FC_NONE;
+ aq_link_eee_t eee = AQ_EEE_DISABLE;
+
+ if (IFM_TYPE(sc->sc_media.ifm_media) != IFM_ETHER)
+ return EINVAL;
+
+ switch (IFM_SUBTYPE(sc->sc_media.ifm_media)) {
+ case IFM_AUTO:
+ rate = AQ_LINK_AUTO;
+ break;
+ case IFM_NONE:
+ rate = AQ_LINK_NONE;
+ break;
+ case IFM_100_TX:
+ rate = AQ_LINK_100M;
+ break;
+ case IFM_1000_T:
+ rate = AQ_LINK_1G;
+ break;
+ case IFM_2500_T:
+ rate = AQ_LINK_2G5;
+ break;
+ case IFM_5000_T:
+ rate = AQ_LINK_5G;
+ break;
+ case IFM_10G_T:
+ rate = AQ_LINK_10G;
+ break;
+ default:
+ device_printf(sc->sc_dev, "unknown media: 0x%X\n",
+ IFM_SUBTYPE(sc->sc_media.ifm_media));
+ return ENODEV;
+ }
+
+ if (sc->sc_media.ifm_media & IFM_FLOW)
+ fc = AQ_FC_ALL;
+
+ /* XXX: todo EEE */
+
+ /* re-initialize hardware with new parameters */
+ aq_set_linkmode(sc, rate, fc, eee);
+
+ return 0;
+}
+
+static void
+aq_ifmedia_status(struct ifnet * const ifp, struct ifmediareq *ifmr)
+{
+ struct aq_softc *sc = ifp->if_softc;
+
+ ifmr->ifm_active = IFM_ETHER;
+ ifmr->ifm_status = IFM_AVALID;
+
+ if (sc->sc_link_rate != AQ_LINK_NONE)
+ ifmr->ifm_status |= IFM_ACTIVE;
+
+ ifmr->ifm_active |= sc->sc_media_active;
+}
+
+static void
+aq_initmedia(struct aq_softc *sc)
+{
+#define IFMEDIA_ETHER_ADD(sc, media) \
+ ifmedia_add(&(sc)->sc_media, IFM_ETHER | media, 0, NULL);
+
+ IFMEDIA_ETHER_ADD(sc, IFM_NONE);
+ if (sc->sc_available_rates & AQ_LINK_100M) {
+ IFMEDIA_ETHER_ADD(sc, IFM_100_TX);
+ IFMEDIA_ETHER_ADD(sc, IFM_100_TX | IFM_FLOW);
+ IFMEDIA_ETHER_ADD(sc, IFM_100_TX | IFM_FDX | IFM_FLOW);
+ }
+ if (sc->sc_available_rates & AQ_LINK_1G) {
+ IFMEDIA_ETHER_ADD(sc, IFM_1000_T | IFM_FDX);
+ IFMEDIA_ETHER_ADD(sc, IFM_1000_T | IFM_FDX | IFM_FLOW);
+ }
+ if (sc->sc_available_rates & AQ_LINK_2G5) {
+ IFMEDIA_ETHER_ADD(sc, IFM_2500_T | IFM_FDX);
+ IFMEDIA_ETHER_ADD(sc, IFM_2500_T | IFM_FDX | IFM_FLOW);
+ }
+ if (sc->sc_available_rates & AQ_LINK_5G) {
+ IFMEDIA_ETHER_ADD(sc, IFM_5000_T | IFM_FDX);
+ IFMEDIA_ETHER_ADD(sc, IFM_5000_T | IFM_FDX | IFM_FLOW);
+ }
+ if (sc->sc_available_rates & AQ_LINK_10G) {
+ IFMEDIA_ETHER_ADD(sc, IFM_10G_T | IFM_FDX);
+ IFMEDIA_ETHER_ADD(sc, IFM_10G_T | IFM_FDX | IFM_FLOW);
+ }
+ IFMEDIA_ETHER_ADD(sc, IFM_AUTO);
+ IFMEDIA_ETHER_ADD(sc, IFM_AUTO | IFM_FLOW);
+
+ /* default: auto without flowcontrol */
+ ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO);
+ aq_set_linkmode(sc, AQ_LINK_AUTO, AQ_FC_NONE, AQ_EEE_DISABLE);
+}
+
+static int
+aq_set_linkmode(struct aq_softc *sc, aq_link_speed_t speed, aq_link_fc_t fc,
+ aq_link_eee_t eee)
+{
+ return sc->sc_fw_ops->set_mode(sc, MPI_INIT, speed, fc, eee);
+}
+
+static int
+aq_get_linkmode(struct aq_softc *sc, aq_link_speed_t *speed, aq_link_fc_t *fc,
+ aq_link_eee_t *eee)
+{
+ aq_hw_fw_mpi_state_t mode;
+ int error;
+
+ error = sc->sc_fw_ops->get_mode(sc, &mode, speed, fc, eee);
+ if (error != 0)
+ return error;
+ if (mode != MPI_INIT)
+ return ENXIO;
+
+ return 0;
+}
+
+static void
+aq_hw_init_tx_path(struct aq_softc *sc)
+{
+ /* Tx TC/RSS number config */
+ AQ_WRITE_REG_BIT(sc, TPB_TX_BUF_REG, TPB_TX_BUF_TC_MODE_EN, 1);
+
+ AQ_WRITE_REG_BIT(sc, THM_LSO_TCP_FLAG1_REG,
+ THM_LSO_TCP_FLAG1_FIRST, 0x0ff6);
+ AQ_WRITE_REG_BIT(sc, THM_LSO_TCP_FLAG1_REG,
+ THM_LSO_TCP_FLAG1_MID, 0x0ff6);
+ AQ_WRITE_REG_BIT(sc, THM_LSO_TCP_FLAG2_REG,
+ THM_LSO_TCP_FLAG2_LAST, 0x0f7f);
+
+ /* misc */
+ AQ_WRITE_REG(sc, TX_TPO2_REG,
+ (sc->sc_features & FEATURES_TPO2) ? TX_TPO2_EN : 0);
+ AQ_WRITE_REG_BIT(sc, TDM_DCA_REG, TDM_DCA_EN, 0);
+ AQ_WRITE_REG_BIT(sc, TDM_DCA_REG, TDM_DCA_MODE, 0);
+
+ AQ_WRITE_REG_BIT(sc, TPB_TX_BUF_REG, TPB_TX_BUF_SCP_INS_EN, 1);
+}
+
+static void
+aq_hw_init_rx_path(struct aq_softc *sc)
+{
+ int i;
+
+ /* clear setting */
+ AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_TC_MODE, 0);
+ AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_FC_MODE, 0);
+ AQ_WRITE_REG(sc, RX_FLR_RSS_CONTROL1_REG, 0);
+ for (i = 0; i < 32; i++) {
+ AQ_WRITE_REG_BIT(sc, RPF_ETHERTYPE_FILTER_REG(i),
+ RPF_ETHERTYPE_FILTER_EN, 0);
+ }
+
+ if (sc->sc_rss_enable) {
+ /* Rx TC/RSS number config */
+ AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_TC_MODE, 1);
+
+ /* Rx flow control */
+ AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_FC_MODE, 1);
+
+ /* RSS Ring selection */
+ switch (sc->sc_nqueues) {
+ case 2:
+ AQ_WRITE_REG(sc, RX_FLR_RSS_CONTROL1_REG,
+ RX_FLR_RSS_CONTROL1_EN | 0x11111111);
+ break;
+ case 4:
+ AQ_WRITE_REG(sc, RX_FLR_RSS_CONTROL1_REG,
+ RX_FLR_RSS_CONTROL1_EN | 0x22222222);
+ break;
+ case 8:
+ AQ_WRITE_REG(sc, RX_FLR_RSS_CONTROL1_REG,
+ RX_FLR_RSS_CONTROL1_EN | 0x33333333);
+ break;
+ }
+ }
+
+ /* L2 and Multicast filters */
+ for (i = 0; i < AQ_HW_MAC_NUM; i++) {
+ AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(i), RPF_L2UC_MSW_EN, 0);
+ AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(i), RPF_L2UC_MSW_ACTION,
+ RPF_ACTION_HOST);
+ }
+ AQ_WRITE_REG(sc, RPF_MCAST_FILTER_MASK_REG, 0);
+ AQ_WRITE_REG(sc, RPF_MCAST_FILTER_REG(0), 0x00010fff);
+
+ /* Vlan filters */
+ AQ_WRITE_REG_BIT(sc, RPF_VLAN_TPID_REG, RPF_VLAN_TPID_OUTER,
+ ETHERTYPE_QINQ);
+ AQ_WRITE_REG_BIT(sc, RPF_VLAN_TPID_REG, RPF_VLAN_TPID_INNER,
+ ETHERTYPE_VLAN);
+ AQ_WRITE_REG_BIT(sc, RPF_VLAN_MODE_REG, RPF_VLAN_MODE_PROMISC, 1);
+
+ if (sc->sc_features & FEATURES_REV_B) {
+ AQ_WRITE_REG_BIT(sc, RPF_VLAN_MODE_REG,
+ RPF_VLAN_MODE_ACCEPT_UNTAGGED, 1);
+ AQ_WRITE_REG_BIT(sc, RPF_VLAN_MODE_REG,
+ RPF_VLAN_MODE_UNTAGGED_ACTION, RPF_ACTION_HOST);
+ }
+
+ /* misc */
+ if (sc->sc_features & FEATURES_RPF2)
+ AQ_WRITE_REG(sc, RX_TCP_RSS_HASH_REG, RX_TCP_RSS_HASH_RPF2);
+ else
+ AQ_WRITE_REG(sc, RX_TCP_RSS_HASH_REG, 0);
+
+ /*
+ * XXX: RX_TCP_RSS_HASH_REG:
+ * linux set 0x000f0000
+ * freebsd set 0x000f001e
+ */
+ /* RSS hash type set for IP/TCP */
+ AQ_WRITE_REG_BIT(sc, RX_TCP_RSS_HASH_REG,
+ RX_TCP_RSS_HASH_TYPE, 0x001e);
+
+ AQ_WRITE_REG_BIT(sc, RPF_L2BC_REG, RPF_L2BC_EN, 1);
+ AQ_WRITE_REG_BIT(sc, RPF_L2BC_REG, RPF_L2BC_ACTION, RPF_ACTION_HOST);
+ AQ_WRITE_REG_BIT(sc, RPF_L2BC_REG, RPF_L2BC_THRESHOLD, 0xffff);
+
+ AQ_WRITE_REG_BIT(sc, RX_DMA_DCA_REG, RX_DMA_DCA_EN, 0);
+ AQ_WRITE_REG_BIT(sc, RX_DMA_DCA_REG, RX_DMA_DCA_MODE, 0);
+}
+
+static void
+aq_hw_interrupt_moderation_set(struct aq_softc *sc)
+{
+ int i;
+
+ if (sc->sc_intr_moderation_enable) {
+ unsigned int tx_min, rx_min; /* 0-255 */
+ unsigned int tx_max, rx_max; /* 0-511? */
+
+ switch (sc->sc_link_rate) {
+ case AQ_LINK_100M:
+ tx_min = 0x4f;
+ tx_max = 0xff;
+ rx_min = 0x04;
+ rx_max = 0x50;
+ break;
+ case AQ_LINK_1G:
+ default:
+ tx_min = 0x4f;
+ tx_max = 0xff;
+ rx_min = 0x30;
+ rx_max = 0x80;
+ break;
+ case AQ_LINK_2G5:
+ tx_min = 0x4f;
+ tx_max = 0xff;
+ rx_min = 0x18;
+ rx_max = 0xe0;
+ break;
+ case AQ_LINK_5G:
+ tx_min = 0x4f;
+ tx_max = 0xff;
+ rx_min = 0x0c;
+ rx_max = 0x70;
+ break;
+ case AQ_LINK_10G:
+ tx_min = 0x4f;
+ tx_max = 0x1ff;
+ rx_min = 0x06; /* freebsd use 80 */
+ rx_max = 0x38; /* freebsd use 120 */
+ break;
+ }
+
+ AQ_WRITE_REG_BIT(sc, TX_DMA_INT_DESC_WRWB_EN_REG,
+ TX_DMA_INT_DESC_WRWB_EN, 0);
+ AQ_WRITE_REG_BIT(sc, TX_DMA_INT_DESC_WRWB_EN_REG,
+ TX_DMA_INT_DESC_MODERATE_EN, 1);
+ AQ_WRITE_REG_BIT(sc, RX_DMA_INT_DESC_WRWB_EN_REG,
+ RX_DMA_INT_DESC_WRWB_EN, 0);
+ AQ_WRITE_REG_BIT(sc, RX_DMA_INT_DESC_WRWB_EN_REG,
+ RX_DMA_INT_DESC_MODERATE_EN, 1);
+
+ for (i = 0; i < AQ_RINGS_NUM; i++) {
+ AQ_WRITE_REG(sc, TX_INTR_MODERATION_CTL_REG(i),
+ __SHIFTIN(tx_min, TX_INTR_MODERATION_CTL_MIN) |
+ __SHIFTIN(tx_max, TX_INTR_MODERATION_CTL_MAX) |
+ TX_INTR_MODERATION_CTL_EN);
+ }
+ for (i = 0; i < AQ_RINGS_NUM; i++) {
+ AQ_WRITE_REG(sc, RX_INTR_MODERATION_CTL_REG(i),
+ __SHIFTIN(rx_min, RX_INTR_MODERATION_CTL_MIN) |
+ __SHIFTIN(rx_max, RX_INTR_MODERATION_CTL_MAX) |
+ RX_INTR_MODERATION_CTL_EN);
+ }
+
+ } else {
+ AQ_WRITE_REG_BIT(sc, TX_DMA_INT_DESC_WRWB_EN_REG,
+ TX_DMA_INT_DESC_WRWB_EN, 1);
+ AQ_WRITE_REG_BIT(sc, TX_DMA_INT_DESC_WRWB_EN_REG,
+ TX_DMA_INT_DESC_MODERATE_EN, 0);
+ AQ_WRITE_REG_BIT(sc, RX_DMA_INT_DESC_WRWB_EN_REG,
+ RX_DMA_INT_DESC_WRWB_EN, 1);
+ AQ_WRITE_REG_BIT(sc, RX_DMA_INT_DESC_WRWB_EN_REG,
+ RX_DMA_INT_DESC_MODERATE_EN, 0);
+
+ for (i = 0; i < AQ_RINGS_NUM; i++) {
+ AQ_WRITE_REG(sc, TX_INTR_MODERATION_CTL_REG(i), 0);
+ }
+ for (i = 0; i < AQ_RINGS_NUM; i++) {
+ AQ_WRITE_REG(sc, RX_INTR_MODERATION_CTL_REG(i), 0);
+ }
+ }
+}
+
+static void
+aq_hw_qos_set(struct aq_softc *sc)
+{
+ uint32_t tc = 0;
+ uint32_t buff_size;
+
+ /* TPS Descriptor rate init */
+ AQ_WRITE_REG_BIT(sc, TPS_DESC_RATE_REG, TPS_DESC_RATE_TA_RST, 0);
+ AQ_WRITE_REG_BIT(sc, TPS_DESC_RATE_REG, TPS_DESC_RATE_LIM, 0xa);
+
+ /* TPS VM init */
+ AQ_WRITE_REG_BIT(sc, TPS_DESC_VM_ARB_MODE_REG, TPS_DESC_VM_ARB_MODE, 0);
+
+ /* TPS TC credits init */
+ AQ_WRITE_REG_BIT(sc, TPS_DESC_TC_ARB_MODE_REG, TPS_DESC_TC_ARB_MODE, 0);
+ AQ_WRITE_REG_BIT(sc, TPS_DATA_TC_ARB_MODE_REG, TPS_DATA_TC_ARB_MODE, 0);
+
+ AQ_WRITE_REG_BIT(sc, TPS_DATA_TCT_REG(tc),
+ TPS_DATA_TCT_CREDIT_MAX, 0xfff);
+ AQ_WRITE_REG_BIT(sc, TPS_DATA_TCT_REG(tc),
+ TPS_DATA_TCT_WEIGHT, 0x64);
+ AQ_WRITE_REG_BIT(sc, TPS_DESC_TCT_REG(tc),
+ TPS_DESC_TCT_CREDIT_MAX, 0x50);
+ AQ_WRITE_REG_BIT(sc, TPS_DESC_TCT_REG(tc),
+ TPS_DESC_TCT_WEIGHT, 0x1e);
+
+ /* Tx buf size */
+ tc = 0;
+ buff_size = AQ_HW_TXBUF_MAX;
+ AQ_WRITE_REG_BIT(sc, TPB_TXB_BUFSIZE_REG(tc), TPB_TXB_BUFSIZE,
+ buff_size);
+ AQ_WRITE_REG_BIT(sc, TPB_TXB_THRESH_REG(tc), TPB_TXB_THRESH_HI,
+ (buff_size * (1024 / 32) * 66) / 100);
+ AQ_WRITE_REG_BIT(sc, TPB_TXB_THRESH_REG(tc), TPB_TXB_THRESH_LO,
+ (buff_size * (1024 / 32) * 50) / 100);
+
+ /* QoS Rx buf size per TC */
+ tc = 0;
+ buff_size = AQ_HW_RXBUF_MAX;
+ AQ_WRITE_REG_BIT(sc, RPB_RXB_BUFSIZE_REG(tc), RPB_RXB_BUFSIZE,
+ buff_size);
+ AQ_WRITE_REG_BIT(sc, RPB_RXB_XOFF_REG(tc), RPB_RXB_XOFF_EN, 0);
+ AQ_WRITE_REG_BIT(sc, RPB_RXB_XOFF_REG(tc), RPB_RXB_XOFF_THRESH_HI,
+ (buff_size * (1024 / 32) * 66) / 100);
+ AQ_WRITE_REG_BIT(sc, RPB_RXB_XOFF_REG(tc), RPB_RXB_XOFF_THRESH_LO,
+ (buff_size * (1024 / 32) * 50) / 100);
+
+ /* QoS 802.1p priority -> TC mapping */
+ int i_priority;
+ for (i_priority = 0; i_priority < 8; i_priority++) {
+ AQ_WRITE_REG_BIT(sc, RPF_RPB_RX_TC_UPT_REG,
+ RPF_RPB_RX_TC_UPT_MASK(i_priority), 0);
+ }
+}
+
+/* called once from aq_attach */
+static int
+aq_init_rss(struct aq_softc *sc)
+{
+ CTASSERT(AQ_RSS_HASHKEY_SIZE == RSS_KEYSIZE);
+ uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
+ uint8_t rss_table[AQ_RSS_INDIRECTION_TABLE_MAX];
+ unsigned int i;
+ int error;
+
+ /* initialize rss key */
+ rss_getkey((uint8_t *)rss_key);
+
+ /* hash to ring table */
+ for (i = 0; i < AQ_RSS_INDIRECTION_TABLE_MAX; i++) {
+ rss_table[i] = i % sc->sc_nqueues;
+ }
+
+ /*
+ * set rss key
+ */
+ for (i = 0; i < __arraycount(rss_key); i++) {
+ uint32_t key_data = sc->sc_rss_enable ? ntohl(rss_key[i]) : 0;
+ AQ_WRITE_REG(sc, RPF_RSS_KEY_WR_DATA_REG, key_data);
+ AQ_WRITE_REG_BIT(sc, RPF_RSS_KEY_ADDR_REG,
+ RPF_RSS_KEY_ADDR, __arraycount(rss_key) - 1 - i);
+ AQ_WRITE_REG_BIT(sc, RPF_RSS_KEY_ADDR_REG,
+ RPF_RSS_KEY_WR_EN, 1);
+ WAIT_FOR(AQ_READ_REG_BIT(sc, RPF_RSS_KEY_ADDR_REG,
+ RPF_RSS_KEY_WR_EN) == 0, 1000, 10, &error);
+ if (error != 0) {
+ device_printf(sc->sc_dev, "%s: rss key write timeout\n",
+ __func__);
+ goto rss_set_timeout;
+ }
+ }
+
+ /*
+ * set rss indirection table
+ *
+ * AQ's rss redirect table is consist of 3bit*64 (192bit) packed array.
+ * we'll make it by __BITMAP(3) macros.
+ */
+ __BITMAP_TYPE(, uint16_t, 3 * AQ_RSS_INDIRECTION_TABLE_MAX) bit3x64;
+ __BITMAP_ZERO(&bit3x64);
+
+#define AQ_3BIT_PACKED_ARRAY_SET(bitmap, idx, val) \
+ do { \
+ if (val & 1) { \
+ __BITMAP_SET((idx) * 3, (bitmap)); \
+ } else { \
+ __BITMAP_CLR((idx) * 3, (bitmap)); \
+ } \
+ if (val & 2) { \
+ __BITMAP_SET((idx) * 3 + 1, (bitmap)); \
+ } else { \
+ __BITMAP_CLR((idx) * 3 + 1, (bitmap)); \
+ } \
+ if (val & 4) { \
+ __BITMAP_SET((idx) * 3 + 2, (bitmap)); \
+ } else { \
+ __BITMAP_CLR((idx) * 3 + 2, (bitmap)); \
+ } \
+ } while (0 /* CONSTCOND */)
+
+ for (i = 0; i < AQ_RSS_INDIRECTION_TABLE_MAX; i++) {
+ AQ_3BIT_PACKED_ARRAY_SET(&bit3x64, i, rss_table[i]);
+ }
+
+ /* write 192bit data in steps of 16bit */
+ for (i = 0; i < (int)__arraycount(bit3x64._b); i++) {
+ AQ_WRITE_REG_BIT(sc, RPF_RSS_REDIR_WR_DATA_REG,
+ RPF_RSS_REDIR_WR_DATA, bit3x64._b[i]);
+ AQ_WRITE_REG_BIT(sc, RPF_RSS_REDIR_ADDR_REG,
+ RPF_RSS_REDIR_ADDR, i);
+ AQ_WRITE_REG_BIT(sc, RPF_RSS_REDIR_ADDR_REG,
+ RPF_RSS_REDIR_WR_EN, 1);
+
+ WAIT_FOR(AQ_READ_REG_BIT(sc, RPF_RSS_REDIR_ADDR_REG,
+ RPF_RSS_REDIR_WR_EN) == 0, 1000, 10, &error);
+ if (error != 0)
+ break;
+ }
+
+ rss_set_timeout:
+ return error;
+}
+
+static void
+aq_hw_l3_filter_set(struct aq_softc *sc)
+{
+ int i;
+
+ /* clear all filter */
+ for (i = 0; i < 8; i++) {
+ AQ_WRITE_REG_BIT(sc, RPF_L3_FILTER_REG(i),
+ RPF_L3_FILTER_L4_EN, 0);
+ }
+}
+
+static void
+aq_update_vlan_filters(struct aq_softc *sc)
+{
+ /* XXX: notyet. vlan always promisc */
+ int i;
+
+ for (i = 0; i < RPF_VLAN_MAX_FILTERS; i++) {
+ AQ_WRITE_REG_BIT(sc, RPF_VLAN_FILTER_REG(i),
+ RPF_VLAN_FILTER_EN, 0);
+ AQ_WRITE_REG_BIT(sc, RPF_VLAN_FILTER_REG(i),
+ RPF_VLAN_FILTER_RXQ_EN, 0);
+ }
+ AQ_WRITE_REG_BIT(sc, RPF_VLAN_MODE_REG, RPF_VLAN_MODE_PROMISC, 1);
+}
+
+static int
+aq_hw_init(struct aq_softc *sc)
+{
+ uint32_t v;
+
+ /* Force limit MRRS on RDM/TDM to 2K */
+ v = AQ_READ_REG(sc, AQ_PCI_REG_CONTROL_6_REG);
+ AQ_WRITE_REG(sc, AQ_PCI_REG_CONTROL_6_REG, (v & ~0x0707) | 0x0404);
+
+ /*
+ * TX DMA total request limit. B0 hardware is not capable to
+ * handle more than (8K-MRRS) incoming DMA data.
+ * Value 24 in 256byte units
+ */
+ AQ_WRITE_REG(sc, AQ_HW_TX_DMA_TOTAL_REQ_LIMIT_REG, 24);
+
+ aq_hw_init_tx_path(sc);
+ aq_hw_init_rx_path(sc);
+
+ aq_hw_interrupt_moderation_set(sc);
+
+ aq_set_mac_addr(sc, AQ_HW_MAC_OWN, sc->sc_enaddr.ether_addr_octet);
+ aq_set_linkmode(sc, AQ_LINK_NONE, AQ_FC_NONE, AQ_EEE_DISABLE);
+
+ aq_hw_qos_set(sc);
+
+ /* Enable interrupt */
+ int irqmode;
+ if (sc->sc_msix)
+ irqmode = AQ_INTR_CTRL_IRQMODE_MSIX;
+ else
+ irqmode = AQ_INTR_CTRL_IRQMODE_MSI;
+
+ AQ_WRITE_REG(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_RESET_DIS);
+ AQ_WRITE_REG_BIT(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_MULTIVEC,
+ sc->sc_msix ? 1 : 0);
+ AQ_WRITE_REG_BIT(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_IRQMODE, irqmode);
+
+ AQ_WRITE_REG(sc, AQ_INTR_AUTOMASK_REG, 0xffffffff);
+
+ AQ_WRITE_REG(sc, AQ_GEN_INTR_MAP_REG(0),
+ ((AQ_B0_ERR_INT << 24) | (1 << 31)) |
+ ((AQ_B0_ERR_INT << 16) | (1 << 23))
+ );
+
+ /* link interrupt */
+ if (!sc->sc_msix)
+ sc->sc_linkstat_irq = AQ_LINKSTAT_IRQ;
+ AQ_WRITE_REG(sc, AQ_GEN_INTR_MAP_REG(3),
+ __BIT(7) | sc->sc_linkstat_irq);
+
+ return 0;
+}
+
+static int
+aq_update_link_status(struct aq_softc *sc)
+{
+ struct ifnet *ifp = &sc->sc_ethercom.ec_if;
+ aq_link_speed_t rate = AQ_LINK_NONE;
+ aq_link_fc_t fc = AQ_FC_NONE;
+ aq_link_eee_t eee = AQ_EEE_DISABLE;
+ unsigned int speed;
+ int changed = 0;
+
+ aq_get_linkmode(sc, &rate, &fc, &eee);
+
+ if (sc->sc_link_rate != rate)
+ changed = 1;
+ if (sc->sc_link_fc != fc)
+ changed = 1;
+ if (sc->sc_link_eee != eee)
+ changed = 1;
+
+ if (changed) {
+ switch (rate) {
+ case AQ_LINK_100M:
+ speed = 100;
+ break;
+ case AQ_LINK_1G:
+ speed = 1000;
+ break;
+ case AQ_LINK_2G5:
+ speed = 2500;
+ break;
+ case AQ_LINK_5G:
+ speed = 5000;
+ break;
+ case AQ_LINK_10G:
+ speed = 10000;
+ break;
+ case AQ_LINK_NONE:
+ default:
+ speed = 0;
+ break;
+ }
+
+ if (sc->sc_link_rate == AQ_LINK_NONE) {
+ /* link DOWN -> UP */
+ device_printf(sc->sc_dev, "link is UP: speed=%u\n",
+ speed);
+ if_link_state_change(ifp, LINK_STATE_UP);
+ } else if (rate == AQ_LINK_NONE) {
+ /* link UP -> DOWN */
+ device_printf(sc->sc_dev, "link is DOWN\n");
+ if_link_state_change(ifp, LINK_STATE_DOWN);
+ } else {
+ device_printf(sc->sc_dev,
+ "link mode changed: speed=%u, fc=0x%x, eee=%x\n",
+ speed, fc, eee);
+ }
+
+ sc->sc_link_rate = rate;
+ sc->sc_link_fc = fc;
+ sc->sc_link_eee = eee;
+
+ aq_mediastatus_update(sc);
+
+ /* update interrupt timing according to new link speed */
+ aq_hw_interrupt_moderation_set(sc);
+ }
+
+ return changed;
+}
+
+#ifdef AQ_EVENT_COUNTERS
+static void
+aq_update_statistics(struct aq_softc *sc)
+{
+ int prev = sc->sc_statistics_idx;
+ int cur = prev ^ 1;
+
+ sc->sc_fw_ops->get_stats(sc, &sc->sc_statistics[cur]);
+
+ /*
+ * aq's internal statistics counter is 32bit.
+ * cauculate delta, and add to evcount
+ */
+#define ADD_DELTA(cur, prev, name) \
+ do { \
+ uint32_t n; \
+ n = (uint32_t)(sc->sc_statistics[cur].name - \
+ sc->sc_statistics[prev].name); \
+ if (n != 0) { \
+ AQ_EVCNT_ADD(sc, name, n); \
+ } \
+ } while (/*CONSTCOND*/0);
+
+ ADD_DELTA(cur, prev, uprc);
+ ADD_DELTA(cur, prev, mprc);
+ ADD_DELTA(cur, prev, bprc);
+ ADD_DELTA(cur, prev, prc);
+ ADD_DELTA(cur, prev, erpr);
+ ADD_DELTA(cur, prev, uptc);
+ ADD_DELTA(cur, prev, mptc);
+ ADD_DELTA(cur, prev, bptc);
+ ADD_DELTA(cur, prev, ptc);
+ ADD_DELTA(cur, prev, erpt);
+ ADD_DELTA(cur, prev, mbtc);
+ ADD_DELTA(cur, prev, bbtc);
+ ADD_DELTA(cur, prev, mbrc);
+ ADD_DELTA(cur, prev, bbrc);
+ ADD_DELTA(cur, prev, ubrc);
+ ADD_DELTA(cur, prev, ubtc);
+ ADD_DELTA(cur, prev, dpc);
+ ADD_DELTA(cur, prev, cprc);
+
+ sc->sc_statistics_idx = cur;
+}
+#endif /* AQ_EVENT_COUNTERS */
+
+/* allocate and map one DMA block */
+static int
+_alloc_dma(struct aq_softc *sc, bus_size_t size, bus_size_t *sizep,
+ void **addrp, bus_dmamap_t *mapp, bus_dma_segment_t *seg)
+{
+ int nsegs, error;
+
+ if ((error = bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, seg,
+ 1, &nsegs, 0)) != 0) {
+ aprint_error_dev(sc->sc_dev,
+ "unable to allocate DMA buffer, error=%d\n", error);
+ goto fail_alloc;
+ }
+
+ if ((error = bus_dmamem_map(sc->sc_dmat, seg, 1, size, addrp,
+ BUS_DMA_COHERENT)) != 0) {
+ aprint_error_dev(sc->sc_dev,
+ "unable to map DMA buffer, error=%d\n", error);
+ goto fail_map;
+ }
+
+ if ((error = bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
+ 0, mapp)) != 0) {
+ aprint_error_dev(sc->sc_dev,
+ "unable to create DMA map, error=%d\n", error);
+ goto fail_create;
+ }
+
+ if ((error = bus_dmamap_load(sc->sc_dmat, *mapp, *addrp, size, NULL,
+ 0)) != 0) {
+ aprint_error_dev(sc->sc_dev,
+ "unable to load DMA map, error=%d\n", error);
+ goto fail_load;
+ }
+
+ *sizep = size;
+ return 0;
+
+ fail_load:
+ bus_dmamap_destroy(sc->sc_dmat, *mapp);
+ *mapp = NULL;
+ fail_create:
+ bus_dmamem_unmap(sc->sc_dmat, *addrp, size);
+ *addrp = NULL;
+ fail_map:
+ bus_dmamem_free(sc->sc_dmat, seg, 1);
+ memset(seg, 0, sizeof(*seg));
+ fail_alloc:
+ *sizep = 0;
+ return error;
+}
+
+static void
+_free_dma(struct aq_softc *sc, bus_size_t *sizep, void **addrp,
+ bus_dmamap_t *mapp, bus_dma_segment_t *seg)
+{
+ if (*mapp != NULL) {
+ bus_dmamap_destroy(sc->sc_dmat, *mapp);
+ *mapp = NULL;
+ }
+ if (*addrp != NULL) {
+ bus_dmamem_unmap(sc->sc_dmat, *addrp, *sizep);
+ *addrp = NULL;
+ }
+ if (*sizep != 0) {
+ bus_dmamem_free(sc->sc_dmat, seg, 1);
+ memset(seg, 0, sizeof(*seg));
+ *sizep = 0;
+ }
+}
+
+static int
+aq_txring_alloc(struct aq_softc *sc, struct aq_txring *txring)
+{
+ int i, error;
+
+ /* allocate tx descriptors */
+ error = _alloc_dma(sc, sizeof(aq_tx_desc_t) * AQ_TXD_NUM,
+ &txring->txr_txdesc_size, (void **)&txring->txr_txdesc,
+ &txring->txr_txdesc_dmamap, txring->txr_txdesc_seg);
+ if (error != 0)
+ return error;
+
+ memset(txring->txr_txdesc, 0, sizeof(aq_tx_desc_t) * AQ_TXD_NUM);
+
+ /* fill tx ring with dmamap */
+ for (i = 0; i < AQ_TXD_NUM; i++) {
+#define AQ_MAXDMASIZE (16 * 1024)
+#define AQ_NTXSEGS 32
+ /* XXX: TODO: error check */
+ bus_dmamap_create(sc->sc_dmat, AQ_MAXDMASIZE, AQ_NTXSEGS,
+ AQ_MAXDMASIZE, 0, 0, &txring->txr_mbufs[i].dmamap);
+ }
+ return 0;
+}
+
+static void
+aq_txring_free(struct aq_softc *sc, struct aq_txring *txring)
+{
+ int i;
+
+ _free_dma(sc, &txring->txr_txdesc_size, (void **)&txring->txr_txdesc,
+ &txring->txr_txdesc_dmamap, txring->txr_txdesc_seg);
+
+ for (i = 0; i < AQ_TXD_NUM; i++) {
+ if (txring->txr_mbufs[i].dmamap != NULL) {
+ if (txring->txr_mbufs[i].m != NULL) {
+ bus_dmamap_unload(sc->sc_dmat,
+ txring->txr_mbufs[i].dmamap);
+ m_freem(txring->txr_mbufs[i].m);
+ txring->txr_mbufs[i].m = NULL;
+ }
+ bus_dmamap_destroy(sc->sc_dmat,
+ txring->txr_mbufs[i].dmamap);
+ txring->txr_mbufs[i].dmamap = NULL;
+ }
+ }
+}
+
+static int
+aq_rxring_alloc(struct aq_softc *sc, struct aq_rxring *rxring)
+{
+ int i, error;
+
+ /* allocate rx descriptors */
+ error = _alloc_dma(sc, sizeof(aq_rx_desc_t) * AQ_RXD_NUM,
+ &rxring->rxr_rxdesc_size, (void **)&rxring->rxr_rxdesc,
+ &rxring->rxr_rxdesc_dmamap, rxring->rxr_rxdesc_seg);
+ if (error != 0)
+ return error;
+
+ memset(rxring->rxr_rxdesc, 0, sizeof(aq_rx_desc_t) * AQ_RXD_NUM);
+
+ /* fill rxring with dmamaps */
+ for (i = 0; i < AQ_RXD_NUM; i++) {
+ rxring->rxr_mbufs[i].m = NULL;
+ /* XXX: TODO: error check */
+ bus_dmamap_create(sc->sc_dmat, MCLBYTES, 1, MCLBYTES, 0, 0,
+ &rxring->rxr_mbufs[i].dmamap);
+ }
+ return 0;
+}
+
+static void
+aq_rxdrain(struct aq_softc *sc, struct aq_rxring *rxring)
+{
+ int i;
+
+ /* free all mbufs allocated for RX */
+ for (i = 0; i < AQ_RXD_NUM; i++) {
+ if (rxring->rxr_mbufs[i].m != NULL) {
+ bus_dmamap_unload(sc->sc_dmat,
+ rxring->rxr_mbufs[i].dmamap);
+ m_freem(rxring->rxr_mbufs[i].m);
+ rxring->rxr_mbufs[i].m = NULL;
+ }
+ }
+}
+
+static void
+aq_rxring_free(struct aq_softc *sc, struct aq_rxring *rxring)
+{
+ int i;
+
+ /* free all mbufs and dmamaps */
+ aq_rxdrain(sc, rxring);
+ for (i = 0; i < AQ_RXD_NUM; i++) {
+ if (rxring->rxr_mbufs[i].dmamap != NULL) {
+ bus_dmamap_destroy(sc->sc_dmat,
+ rxring->rxr_mbufs[i].dmamap);
+ rxring->rxr_mbufs[i].dmamap = NULL;
+ }
+ }
+
+ /* free RX descriptor */
+ _free_dma(sc, &rxring->rxr_rxdesc_size, (void **)&rxring->rxr_rxdesc,
+ &rxring->rxr_rxdesc_dmamap, rxring->rxr_rxdesc_seg);
+}
+
+static void
+aq_rxring_setmbuf(struct aq_softc *sc, struct aq_rxring *rxring, int idx,
+ struct mbuf *m)
+{
+ int error;
+
+ /* if mbuf already exists, unload and free */
+ if (rxring->rxr_mbufs[idx].m != NULL) {
+ bus_dmamap_unload(sc->sc_dmat, rxring->rxr_mbufs[idx].dmamap);
+ m_freem(rxring->rxr_mbufs[idx].m);
+ rxring->rxr_mbufs[idx].m = NULL;
+ }
+
+ rxring->rxr_mbufs[idx].m = m;
+
+ m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
+ error = bus_dmamap_load_mbuf(sc->sc_dmat, rxring->rxr_mbufs[idx].dmamap,
+ m, BUS_DMA_READ | BUS_DMA_NOWAIT);
+ if (error) {
+ device_printf(sc->sc_dev,
+ "unable to load rx DMA map %d, error = %d\n", idx, error);
+ panic("%s: unable to load rx DMA map. error=%d",
+ __func__, error);
+ }
+ bus_dmamap_sync(sc->sc_dmat, rxring->rxr_mbufs[idx].dmamap, 0,
+ rxring->rxr_mbufs[idx].dmamap->dm_mapsize, BUS_DMASYNC_PREREAD);
+}
+
+static inline void
+aq_rxring_reset_desc(struct aq_softc *sc, struct aq_rxring *rxring, int idx)
+{
+ /* refill rxdesc, and sync */
+ rxring->rxr_rxdesc[idx].read.buf_addr =
+ htole64(rxring->rxr_mbufs[idx].dmamap->dm_segs[0].ds_addr);
+ rxring->rxr_rxdesc[idx].read.hdr_addr = 0;
+ bus_dmamap_sync(sc->sc_dmat, rxring->rxr_rxdesc_dmamap,
+ sizeof(aq_rx_desc_t) * idx, sizeof(aq_rx_desc_t),
+ BUS_DMASYNC_PREWRITE);
+}
+
+static struct mbuf *
+aq_alloc_mbuf(void)
+{
+ struct mbuf *m;
+
+ MGETHDR(m, M_DONTWAIT, MT_DATA);
+ if (m == NULL)
+ return NULL;
+
+ MCLGET(m, M_DONTWAIT);
+ if ((m->m_flags & M_EXT) == 0) {
+ m_freem(m);
+ return NULL;
+ }
+
+ return m;
+}
+
+/* allocate mbuf and unload dmamap */
+static int
+aq_rxring_add(struct aq_softc *sc, struct aq_rxring *rxring, int idx)
+{
+ struct mbuf *m;
+
+ m = aq_alloc_mbuf();
+ if (m == NULL)
+ return ENOBUFS;
+
+ aq_rxring_setmbuf(sc, rxring, idx, m);
+ return 0;
+}
+
+static int
+aq_txrx_rings_alloc(struct aq_softc *sc)
+{
+ int n, error;
+
+ for (n = 0; n < sc->sc_nqueues; n++) {
+ sc->sc_queue[n].sc = sc;
+ sc->sc_queue[n].txring.txr_sc = sc;
+ sc->sc_queue[n].txring.txr_index = n;
+ mutex_init(&sc->sc_queue[n].txring.txr_mutex, MUTEX_DEFAULT,
+ IPL_NET);
+ error = aq_txring_alloc(sc, &sc->sc_queue[n].txring);
+ if (error != 0)
+ goto failure;
+
+ error = aq_tx_pcq_alloc(sc, &sc->sc_queue[n].txring);
+ if (error != 0)
+ goto failure;
+
+ sc->sc_queue[n].rxring.rxr_sc = sc;
+ sc->sc_queue[n].rxring.rxr_index = n;
+ mutex_init(&sc->sc_queue[n].rxring.rxr_mutex, MUTEX_DEFAULT,
+ IPL_NET);
+ error = aq_rxring_alloc(sc, &sc->sc_queue[n].rxring);
+ if (error != 0)
+ break;
+ }
+
+ failure:
+ return error;
+}
+
+static void
+aq_txrx_rings_free(struct aq_softc *sc)
+{
+ int n;
+
+ for (n = 0; n < sc->sc_nqueues; n++) {
+ aq_txring_free(sc, &sc->sc_queue[n].txring);
+ mutex_destroy(&sc->sc_queue[n].txring.txr_mutex);
+
+ aq_tx_pcq_free(sc, &sc->sc_queue[n].txring);
+
+ aq_rxring_free(sc, &sc->sc_queue[n].rxring);
+ mutex_destroy(&sc->sc_queue[n].rxring.rxr_mutex);
+ }
+}
+
+static int
+aq_tx_pcq_alloc(struct aq_softc *sc, struct aq_txring *txring)
+{
+ int error = 0;
+ txring->txr_softint = NULL;
+
+ txring->txr_pcq = pcq_create(AQ_TXD_NUM, KM_NOSLEEP);
+ if (txring->txr_pcq == NULL) {
+ aprint_error_dev(sc->sc_dev,
+ "unable to allocate pcq for TXring[%d]\n",
+ txring->txr_index);
+ error = ENOMEM;
+ goto done;
+ }
+
+ txring->txr_softint = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE,
+ aq_deferred_transmit, txring);
+ if (txring->txr_softint == NULL) {
+ aprint_error_dev(sc->sc_dev,
+ "unable to establish softint for TXring[%d]\n",
+ txring->txr_index);
+ error = ENOENT;
+ }
+
+ done:
+ return error;
+}
+
+static void
+aq_tx_pcq_free(struct aq_softc *sc, struct aq_txring *txring)
+{
+ struct mbuf *m;
+
+ if (txring->txr_softint != NULL) {
+ softint_disestablish(txring->txr_softint);
+ txring->txr_softint = NULL;
+ }
+
+ if (txring->txr_pcq != NULL) {
+ while ((m = pcq_get(txring->txr_pcq)) != NULL)
+ m_freem(m);
+ pcq_destroy(txring->txr_pcq);
+ txring->txr_pcq = NULL;
+ }
+}
+
+static void
+aq_tick(void *arg)
+{
+ struct aq_softc *sc = arg;
+
+ if (sc->sc_poll_linkstat || sc->sc_detect_linkstat) {
+ sc->sc_detect_linkstat = false;
+ aq_update_link_status(sc);
+ }
+
+#ifdef AQ_EVENT_COUNTERS
+ if (sc->sc_poll_statistics)
+ aq_update_statistics(sc);
+#endif
+
+ if (sc->sc_poll_linkstat
+#ifdef AQ_EVENT_COUNTERS
+ || sc->sc_poll_statistics
+#endif
+ ) {
+ callout_schedule(&sc->sc_tick_ch, hz);
+ }
+}
+
+/* interrupt enable/disable */
+static void
+aq_enable_intr(struct aq_softc *sc, bool link, bool txrx)
+{
+ uint32_t imask = 0;
+ int i;
+
+ if (txrx) {
+ for (i = 0; i < sc->sc_nqueues; i++) {
+ imask |= __BIT(sc->sc_tx_irq[i]);
+ imask |= __BIT(sc->sc_rx_irq[i]);
+ }
+ }
+
+ if (link)
+ imask |= __BIT(sc->sc_linkstat_irq);
+
+ AQ_WRITE_REG(sc, AQ_INTR_MASK_REG, imask);
+ AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG, 0xffffffff);
+}
+
+static int
+aq_legacy_intr(void *arg)
+{
+ struct aq_softc *sc = arg;
+ uint32_t status;
+ int nintr = 0;
+
+ status = AQ_READ_REG(sc, AQ_INTR_STATUS_REG);
+ AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG, 0xffffffff);
+
+ if (status & __BIT(sc->sc_linkstat_irq)) {
+ sc->sc_detect_linkstat = true;
+ callout_schedule(&sc->sc_tick_ch, 0);
+ nintr++;
+ }
+
+ if (status & __BIT(sc->sc_rx_irq[0])) {
+ nintr += aq_rx_intr(&sc->sc_queue[0].rxring);
+ }
+
+ if (status & __BIT(sc->sc_tx_irq[0])) {
+ nintr += aq_tx_intr(&sc->sc_queue[0].txring);
+ }
+
+ return nintr;
+}
+
+static int
+aq_txrx_intr(void *arg)
+{
+ struct aq_queue *queue = arg;
+ struct aq_softc *sc = queue->sc;
+ struct aq_txring *txring = &queue->txring;
+ struct aq_rxring *rxring = &queue->rxring;
+ uint32_t status;
+ int nintr = 0;
+ int txringidx, rxringidx, txirq, rxirq;
+
+ txringidx = txring->txr_index;
+ rxringidx = rxring->rxr_index;
+ txirq = sc->sc_tx_irq[txringidx];
+ rxirq = sc->sc_rx_irq[rxringidx];
+
+ status = AQ_READ_REG(sc, AQ_INTR_STATUS_REG);
+ if ((status & (__BIT(txirq) | __BIT(rxirq))) == 0) {
+ /* stray interrupt? */
+ return 0;
+ }
+
+ nintr += aq_rx_intr(rxring);
+ nintr += aq_tx_intr(txring);
+
+ return nintr;
+}
+
+static int
+aq_link_intr(void *arg)
+{
+ struct aq_softc *sc = arg;
+ uint32_t status;
+ int nintr = 0;
+
+ status = AQ_READ_REG(sc, AQ_INTR_STATUS_REG);
+ if (status & __BIT(sc->sc_linkstat_irq)) {
+ sc->sc_detect_linkstat = true;
+ callout_schedule(&sc->sc_tick_ch, 0);
+ AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG,
+ __BIT(sc->sc_linkstat_irq));
+ nintr++;
+ }
+
+ return nintr;
+}
+
+static void
+aq_txring_reset(struct aq_softc *sc, struct aq_txring *txring, bool start)
+{
+ const int ringidx = txring->txr_index;
+ int i;
+
+ mutex_enter(&txring->txr_mutex);
+
+ txring->txr_prodidx = 0;
+ txring->txr_considx = 0;
+ txring->txr_nfree = AQ_TXD_NUM;
+ txring->txr_active = false;
+
+ /* free mbufs untransmitted */
+ for (i = 0; i < AQ_TXD_NUM; i++) {
+ if (txring->txr_mbufs[i].m != NULL) {
+ m_freem(txring->txr_mbufs[i].m);
+ txring->txr_mbufs[i].m = NULL;
+ }
+ }
+
+ /* disable DMA */
+ AQ_WRITE_REG_BIT(sc, TX_DMA_DESC_REG(ringidx), TX_DMA_DESC_EN, 0);
+
+ if (start) {
+ /* TX descriptor physical address */
+ paddr_t paddr = txring->txr_txdesc_dmamap->dm_segs[0].ds_addr;
+ AQ_WRITE_REG(sc, TX_DMA_DESC_BASE_ADDRLSW_REG(ringidx), paddr);
+ AQ_WRITE_REG(sc, TX_DMA_DESC_BASE_ADDRMSW_REG(ringidx),
+ (uint32_t)((uint64_t)paddr >> 32));
+
+ /* TX descriptor size */
+ AQ_WRITE_REG_BIT(sc, TX_DMA_DESC_REG(ringidx), TX_DMA_DESC_LEN,
+ AQ_TXD_NUM / 8);
+
+ /* reload TAIL pointer */
+ txring->txr_prodidx = txring->txr_considx =
+ AQ_READ_REG(sc, TX_DMA_DESC_TAIL_PTR_REG(ringidx));
+ AQ_WRITE_REG(sc, TX_DMA_DESC_WRWB_THRESH_REG(ringidx), 0);
+
+ /* Mapping interrupt vector */
+ AQ_WRITE_REG_BIT(sc, AQ_INTR_IRQ_MAP_TX_REG(ringidx),
+ AQ_INTR_IRQ_MAP_TX_IRQMAP(ringidx), sc->sc_tx_irq[ringidx]);
+ AQ_WRITE_REG_BIT(sc, AQ_INTR_IRQ_MAP_TX_REG(ringidx),
+ AQ_INTR_IRQ_MAP_TX_EN(ringidx), true);
+
+ /* enable DMA */
+ AQ_WRITE_REG_BIT(sc, TX_DMA_DESC_REG(ringidx),
+ TX_DMA_DESC_EN, 1);
+
+ const int cpuid = 0; /* XXX? */
+ AQ_WRITE_REG_BIT(sc, TDM_DCAD_REG(ringidx),
+ TDM_DCAD_CPUID, cpuid);
+ AQ_WRITE_REG_BIT(sc, TDM_DCAD_REG(ringidx),
+ TDM_DCAD_CPUID_EN, 0);
+
+ txring->txr_active = true;
+ }
+
+ mutex_exit(&txring->txr_mutex);
+}
+
+static int
+aq_rxring_reset(struct aq_softc *sc, struct aq_rxring *rxring, bool start)
+{
+ const int ringidx = rxring->rxr_index;
+ int i;
+ int error = 0;
+
+ mutex_enter(&rxring->rxr_mutex);
+ rxring->rxr_active = false;
+
+ /* disable DMA */
+ AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx), RX_DMA_DESC_EN, 0);
+
+ /* free all RX mbufs */
+ aq_rxdrain(sc, rxring);
+
+ if (start) {
+ for (i = 0; i < AQ_RXD_NUM; i++) {
+ error = aq_rxring_add(sc, rxring, i);
+ if (error != 0) {
+ aq_rxdrain(sc, rxring);
+ return error;
+ }
+ aq_rxring_reset_desc(sc, rxring, i);
+ }
+
+ /* RX descriptor physical address */
+ paddr_t paddr = rxring->rxr_rxdesc_dmamap->dm_segs[0].ds_addr;
+ AQ_WRITE_REG(sc, RX_DMA_DESC_BASE_ADDRLSW_REG(ringidx), paddr);
+ AQ_WRITE_REG(sc, RX_DMA_DESC_BASE_ADDRMSW_REG(ringidx),
+ (uint32_t)((uint64_t)paddr >> 32));
+
+ /* RX descriptor size */
+ AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx), RX_DMA_DESC_LEN,
+ AQ_RXD_NUM / 8);
+
+ /* maximum receive frame size */
+ AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_BUFSIZE_REG(ringidx),
+ RX_DMA_DESC_BUFSIZE_DATA, MCLBYTES / 1024);
+ AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_BUFSIZE_REG(ringidx),
+ RX_DMA_DESC_BUFSIZE_HDR, 0 / 1024);
+
+ AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx),
+ RX_DMA_DESC_HEADER_SPLIT, 0);
+ AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx),
+ RX_DMA_DESC_VLAN_STRIP,
+ (sc->sc_ethercom.ec_capenable & ETHERCAP_VLAN_HWTAGGING) ?
+ 1 : 0);
+
+ /*
+ * reload TAIL pointer, and update readidx
+ * (HEAD pointer cannot write)
+ */
+ rxring->rxr_readidx = AQ_READ_REG_BIT(sc,
+ RX_DMA_DESC_HEAD_PTR_REG(ringidx), RX_DMA_DESC_HEAD_PTR);
+ AQ_WRITE_REG(sc, RX_DMA_DESC_TAIL_PTR_REG(ringidx),
+ (rxring->rxr_readidx + AQ_RXD_NUM - 1) % AQ_RXD_NUM);
+
+ /* Rx ring set mode */
+
+ /* Mapping interrupt vector */
+ AQ_WRITE_REG_BIT(sc, AQ_INTR_IRQ_MAP_RX_REG(ringidx),
+ AQ_INTR_IRQ_MAP_RX_IRQMAP(ringidx), sc->sc_rx_irq[ringidx]);
+ AQ_WRITE_REG_BIT(sc, AQ_INTR_IRQ_MAP_RX_REG(ringidx),
+ AQ_INTR_IRQ_MAP_RX_EN(ringidx), 1);
+
+ const int cpuid = 0; /* XXX? */
+ AQ_WRITE_REG_BIT(sc, RX_DMA_DCAD_REG(ringidx),
+ RX_DMA_DCAD_CPUID, cpuid);
+ AQ_WRITE_REG_BIT(sc, RX_DMA_DCAD_REG(ringidx),
+ RX_DMA_DCAD_DESC_EN, 0);
+ AQ_WRITE_REG_BIT(sc, RX_DMA_DCAD_REG(ringidx),
+ RX_DMA_DCAD_HEADER_EN, 0);
+ AQ_WRITE_REG_BIT(sc, RX_DMA_DCAD_REG(ringidx),
+ RX_DMA_DCAD_PAYLOAD_EN, 0);
+
+ /* enable DMA. start receiving */
+ AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx),
+ RX_DMA_DESC_EN, 1);
+
+ rxring->rxr_active = true;
+ }
+
+ mutex_exit(&rxring->rxr_mutex);
+ return error;
+}
+
+#define TXRING_NEXTIDX(idx) \
+ (((idx) >= (AQ_TXD_NUM - 1)) ? 0 : ((idx) + 1))
+#define RXRING_NEXTIDX(idx) \
+ (((idx) >= (AQ_RXD_NUM - 1)) ? 0 : ((idx) + 1))
+
+static int
+aq_encap_txring(struct aq_softc *sc, struct aq_txring *txring, struct mbuf **mp)
+{
+ bus_dmamap_t map;
+ struct mbuf *m = *mp;
+ uint32_t ctl1, ctl1_ctx, ctl2;
+ int idx, i, error;
+
+ idx = txring->txr_prodidx;
+ map = txring->txr_mbufs[idx].dmamap;
+
+ error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m,
+ BUS_DMA_WRITE | BUS_DMA_NOWAIT);
+ if (error == EFBIG) {
+ struct mbuf *n;
+ n = m_defrag(m, M_DONTWAIT);
+ if (n == NULL)
+ return EFBIG;
+ /* m_defrag() preserve m */
+ KASSERT(n == m);
+ error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m,
+ BUS_DMA_WRITE | BUS_DMA_NOWAIT);
+ }
+ if (error != 0)
+ return error;
+
+ /*
+ * check spaces of free descriptors.
+ * +1 is additional descriptor for context (vlan, etc,.)
+ */
+ if ((map->dm_nsegs + 1) > txring->txr_nfree) {
+ device_printf(sc->sc_dev,
+ "TX: not enough descriptors left %d for %d segs\n",
+ txring->txr_nfree, map->dm_nsegs + 1);
+ bus_dmamap_unload(sc->sc_dmat, map);
+ return ENOBUFS;
+ }
+
+ /* sync dma for mbuf */
+ bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
+ BUS_DMASYNC_PREWRITE);
+
+ ctl1_ctx = 0;
+ ctl2 = __SHIFTIN(m->m_pkthdr.len, AQ_TXDESC_CTL2_LEN);
+
+ if (vlan_has_tag(m)) {
+ ctl1 = AQ_TXDESC_CTL1_TYPE_TXC;
+ ctl1 |= __SHIFTIN(vlan_get_tag(m), AQ_TXDESC_CTL1_VID);
+
+ ctl1_ctx |= AQ_TXDESC_CTL1_CMD_VLAN;
+ ctl2 |= AQ_TXDESC_CTL2_CTX_EN;
+
+ /* fill context descriptor and forward index */
+ txring->txr_txdesc[idx].buf_addr = 0;
+ txring->txr_txdesc[idx].ctl1 = htole32(ctl1);
+ txring->txr_txdesc[idx].ctl2 = 0;
+
+ idx = TXRING_NEXTIDX(idx);
+ txring->txr_nfree--;
+ }
+
+ if (m->m_pkthdr.csum_flags & M_CSUM_IPv4)
+ ctl1_ctx |= AQ_TXDESC_CTL1_CMD_IP4CSUM;
+ if (m->m_pkthdr.csum_flags &
+ (M_CSUM_TCPv4 | M_CSUM_UDPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv6)) {
+ ctl1_ctx |= AQ_TXDESC_CTL1_CMD_L4CSUM;
+ }
+
+ /* fill descriptor(s) */
+ for (i = 0; i < map->dm_nsegs; i++) {
+ ctl1 = ctl1_ctx | AQ_TXDESC_CTL1_TYPE_TXD |
+ __SHIFTIN(map->dm_segs[i].ds_len, AQ_TXDESC_CTL1_BLEN);
+ ctl1 |= AQ_TXDESC_CTL1_CMD_FCS;
+
+ if (i == 0) {
+ /* remember mbuf of these descriptors */
+ txring->txr_mbufs[idx].m = m;
+ } else {
+ txring->txr_mbufs[idx].m = NULL;
+ }
+
+ if (i == map->dm_nsegs - 1) {
+ /* last segment, mark an EndOfPacket, and cause intr */
+ ctl1 |= AQ_TXDESC_CTL1_EOP | AQ_TXDESC_CTL1_CMD_WB;
+ }
+
+ txring->txr_txdesc[idx].buf_addr =
+ htole64(map->dm_segs[i].ds_addr);
+ txring->txr_txdesc[idx].ctl1 = htole32(ctl1);
+ txring->txr_txdesc[idx].ctl2 = htole32(ctl2);
+
+ bus_dmamap_sync(sc->sc_dmat, txring->txr_txdesc_dmamap,
+ sizeof(aq_tx_desc_t) * idx, sizeof(aq_tx_desc_t),
+ BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+
+ idx = TXRING_NEXTIDX(idx);
+ txring->txr_nfree--;
+ }
+
+ txring->txr_prodidx = idx;
+
+ return 0;
+}
+
+static int
+aq_tx_intr(void *arg)
+{
+ struct aq_txring *txring = arg;
+ struct aq_softc *sc = txring->txr_sc;
+ struct ifnet *ifp = &sc->sc_ethercom.ec_if;
+ const int ringidx = txring->txr_index;
+ unsigned int idx, hw_head, n = 0;
+
+ mutex_enter(&txring->txr_mutex);
+
+ if (!txring->txr_active)
+ goto tx_intr_done;
+
+ hw_head = AQ_READ_REG_BIT(sc, TX_DMA_DESC_HEAD_PTR_REG(ringidx),
+ TX_DMA_DESC_HEAD_PTR);
+ if (hw_head == txring->txr_considx) {
+ goto tx_intr_done;
+ }
+
+ for (idx = txring->txr_considx; idx != hw_head;
+ idx = TXRING_NEXTIDX(idx), n++) {
+
+ if (txring->txr_mbufs[idx].m != NULL) {
+ bus_dmamap_unload(sc->sc_dmat,
+ txring->txr_mbufs[idx].dmamap);
+ m_freem(txring->txr_mbufs[idx].m);
+ txring->txr_mbufs[idx].m = NULL;
+ ifp->if_opackets++;
+ }
+
+ txring->txr_nfree++;
+ }
+ txring->txr_considx = idx;
+
+ if (ringidx == 0 && txring->txr_nfree >= AQ_TXD_MIN)
+ ifp->if_flags &= ~IFF_OACTIVE;
+
+ /* no more pending TX packet, cancel watchdog */
+ if (txring->txr_nfree >= AQ_TXD_NUM)
+ ifp->if_timer = 0;
+
+ tx_intr_done:
+ mutex_exit(&txring->txr_mutex);
+
+ AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG, __BIT(sc->sc_tx_irq[ringidx]));
+ return n;
+}
+
+static int
+aq_rx_intr(void *arg)
+{
+ struct aq_rxring *rxring = arg;
+ struct aq_softc *sc = rxring->rxr_sc;
+ struct ifnet *ifp = &sc->sc_ethercom.ec_if;
+ const int ringidx = rxring->rxr_index;
+ aq_rx_desc_t *rxd;
+ struct mbuf *m, *m0, *mprev, *new_m;
+ uint32_t rxd_type, rxd_hash __unused;
+ uint16_t rxd_status, rxd_pktlen;
+ uint16_t rxd_nextdescptr __unused, rxd_vlan __unused;
+ unsigned int idx, n = 0;
+
+ mutex_enter(&rxring->rxr_mutex);
+
+ if (!rxring->rxr_active)
+ goto rx_intr_done;
+
+ if (rxring->rxr_readidx == AQ_READ_REG_BIT(sc,
+ RX_DMA_DESC_HEAD_PTR_REG(ringidx), RX_DMA_DESC_HEAD_PTR)) {
+ goto rx_intr_done;
+ }
+
+ m0 = mprev = NULL;
+ for (idx = rxring->rxr_readidx;
+ idx != AQ_READ_REG_BIT(sc, RX_DMA_DESC_HEAD_PTR_REG(ringidx),
+ RX_DMA_DESC_HEAD_PTR); idx = RXRING_NEXTIDX(idx), n++) {
+
+ bus_dmamap_sync(sc->sc_dmat, rxring->rxr_rxdesc_dmamap,
+ sizeof(aq_rx_desc_t) * idx, sizeof(aq_rx_desc_t),
+ BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+
+ rxd = &rxring->rxr_rxdesc[idx];
+ rxd_status = le16toh(rxd->wb.status);
+
+ if ((rxd_status & RXDESC_STATUS_DD) == 0)
+ break; /* not yet done */
+
+ rxd_type = le32toh(rxd->wb.type);
+ rxd_pktlen = le16toh(rxd->wb.pkt_len);
+ rxd_nextdescptr = le16toh(rxd->wb.next_desc_ptr);
+ rxd_hash = le32toh(rxd->wb.rss_hash);
+ rxd_vlan = le16toh(rxd->wb.vlan);
+
+ if ((rxd_status & RXDESC_STATUS_MACERR) ||
+ (rxd_type & RXDESC_TYPE_MAC_DMA_ERR)) {
+ ifp->if_ierrors++;
+ goto rx_next;
+ }
+
+ bus_dmamap_sync(sc->sc_dmat, rxring->rxr_mbufs[idx].dmamap, 0,
+ rxring->rxr_mbufs[idx].dmamap->dm_mapsize,
+ BUS_DMASYNC_POSTREAD);
+ m = rxring->rxr_mbufs[idx].m;
+
+ new_m = aq_alloc_mbuf();
+ if (new_m == NULL) {
+ /*
+ * cannot allocate new mbuf.
+ * discard this packet, and reuse mbuf for next.
+ */
+ goto rx_next;
+ }
+ rxring->rxr_mbufs[idx].m = NULL;
+ aq_rxring_setmbuf(sc, rxring, idx, new_m);
+
+ if (m0 == NULL) {
+ m0 = m;
+ } else {
+ if (m->m_flags & M_PKTHDR)
+ m_remove_pkthdr(m);
+ mprev->m_next = m;
+ }
+ mprev = m;
+
+ if ((rxd_status & RXDESC_STATUS_EOP) == 0) {
+ m->m_len = MCLBYTES;
+ } else {
+ /* last buffer */
+ m->m_len = rxd_pktlen % MCLBYTES;
+ m0->m_pkthdr.len = rxd_pktlen;
+ /* VLAN offloading */
+ if ((sc->sc_ethercom.ec_capenable &
+ ETHERCAP_VLAN_HWTAGGING) &&
+ (__SHIFTOUT(rxd_type, RXDESC_TYPE_PKTTYPE_VLAN) ||
+ __SHIFTOUT(rxd_type,
+ RXDESC_TYPE_PKTTYPE_VLAN_DOUBLE))) {
+ vlan_set_tag(m0, rxd_vlan);
+ }
+
+ /* Checksum offloading */
+ unsigned int pkttype_eth =
+ __SHIFTOUT(rxd_type, RXDESC_TYPE_PKTTYPE_ETHER);
+ if ((ifp->if_capabilities & IFCAP_CSUM_IPv4_Rx) &&
+ (pkttype_eth == RXDESC_TYPE_PKTTYPE_ETHER_IPV4) &&
+ __SHIFTOUT(rxd_type,
+ RXDESC_TYPE_IPV4_CSUM_CHECKED)) {
+ m0->m_pkthdr.csum_flags |= M_CSUM_IPv4;
+ if (__SHIFTOUT(rxd_status,
+ RXDESC_STATUS_IPV4_CSUM_NG))
+ m0->m_pkthdr.csum_flags |=
+ M_CSUM_IPv4_BAD;
+ }
+#if notyet
+ /*
+ * XXX: aq always marks BAD for fragmented packet.
+ * we should peek L3 header, and ignore cksum flags
+ * if the packet is fragmented.
+ */
+ if (__SHIFTOUT(rxd_type,
+ RXDESC_TYPE_TCPUDP_CSUM_CHECKED)) {
+ bool checked = false;
+ unsigned int pkttype_proto =
+ __SHIFTOUT(rxd_type,
+ RXDESC_TYPE_PKTTYPE_PROTO);
+
+ if (pkttype_proto ==
+ RXDESC_TYPE_PKTTYPE_PROTO_TCP) {
+ if ((pkttype_eth ==
+ RXDESC_TYPE_PKTTYPE_ETHER_IPV4) &&
+ (ifp->if_capabilities &
+ IFCAP_CSUM_TCPv4_Rx)) {
+ m0->m_pkthdr.csum_flags |=
+ M_CSUM_TCPv4;
+ checked = true;
+ } else if ((pkttype_eth ==
+ RXDESC_TYPE_PKTTYPE_ETHER_IPV6) &&
+ (ifp->if_capabilities &
+ IFCAP_CSUM_TCPv6_Rx)) {
+ m0->m_pkthdr.csum_flags |=
+ M_CSUM_TCPv6;
+ checked = true;
+ }
+ } else if (pkttype_proto ==
+ RXDESC_TYPE_PKTTYPE_PROTO_UDP) {
+ if ((pkttype_eth ==
+ RXDESC_TYPE_PKTTYPE_ETHER_IPV4) &&
+ (ifp->if_capabilities &
+ IFCAP_CSUM_UDPv4_Rx)) {
+ m0->m_pkthdr.csum_flags |=
+ M_CSUM_UDPv4;
+ checked = true;
+ } else if ((pkttype_eth ==
+ RXDESC_TYPE_PKTTYPE_ETHER_IPV6) &&
+ (ifp->if_capabilities &
+ IFCAP_CSUM_UDPv6_Rx)) {
+ m0->m_pkthdr.csum_flags |=
+ M_CSUM_UDPv6;
+ checked = true;
+ }
+ }
+ if (checked &&
+ (__SHIFTOUT(rxd_status,
+ RXDESC_STATUS_TCPUDP_CSUM_ERROR) ||
+ !__SHIFTOUT(rxd_status,
+ RXDESC_STATUS_TCPUDP_CSUM_OK))) {
+ m0->m_pkthdr.csum_flags |=
+ M_CSUM_TCP_UDP_BAD;
+ }
+ }
+#endif
+
+ m_set_rcvif(m0, ifp);
+ if_percpuq_enqueue(ifp->if_percpuq, m0);
+
+ m0 = mprev = NULL;
+ }
+
+ rx_next:
+ aq_rxring_reset_desc(sc, rxring, idx);
+ AQ_WRITE_REG(sc, RX_DMA_DESC_TAIL_PTR_REG(ringidx), idx);
+ }
+ rxring->rxr_readidx = idx;
+
+ rx_intr_done:
+ mutex_exit(&rxring->rxr_mutex);
+
+ AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG, __BIT(sc->sc_rx_irq[ringidx]));
+ return n;
+}
+
+static int
+aq_ifflags_cb(struct ethercom *ec)
+{
+ struct ifnet *ifp = &ec->ec_if;
+ struct aq_softc *sc = ifp->if_softc;
+ int i, ecchange, error = 0;
+ unsigned short iffchange;
+
+ AQ_LOCK(sc);
+
+ iffchange = ifp->if_flags ^ sc->sc_if_flags;
+ if ((iffchange & IFF_PROMISC) != 0)
+ error = aq_set_filter(sc);
+
+ ecchange = ec->ec_capenable ^ sc->sc_ec_capenable;
+ if (ecchange & ETHERCAP_VLAN_HWTAGGING) {
+ for (i = 0; i < AQ_RINGS_NUM; i++) {
+ AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(i),
+ RX_DMA_DESC_VLAN_STRIP,
+ (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) ?
+ 1 : 0);
+ }
+ }
+
+ sc->sc_ec_capenable = ec->ec_capenable;
+ sc->sc_if_flags = ifp->if_flags;
+
+ AQ_UNLOCK(sc);
+
+ return error;
+}
+
+static int
+aq_init(struct ifnet *ifp)
+{
+ struct aq_softc *sc = ifp->if_softc;
+ int i, error = 0;
+
+ AQ_LOCK(sc);
+
+ aq_update_vlan_filters(sc);
+ aq_set_capability(sc);
+
+ for (i = 0; i < sc->sc_nqueues; i++) {
+ aq_txring_reset(sc, &sc->sc_queue[i].txring, true);
+ }
+
+ /* invalidate RX descriptor cache */
+ AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_CACHE_INIT_REG, RX_DMA_DESC_CACHE_INIT,
+ AQ_READ_REG_BIT(sc,
+ RX_DMA_DESC_CACHE_INIT_REG, RX_DMA_DESC_CACHE_INIT) ^ 1);
+
+ /* start RX */
+ for (i = 0; i < sc->sc_nqueues; i++) {
+ error = aq_rxring_reset(sc, &sc->sc_queue[i].rxring, true);
+ if (error != 0) {
+ device_printf(sc->sc_dev, "%s: cannot allocate rxbuf\n",
+ __func__);
+ goto aq_init_failure;
+ }
+ }
+ aq_init_rss(sc);
+ aq_hw_l3_filter_set(sc);
+
+ /* need to start callout? */
+ if (sc->sc_poll_linkstat
+#ifdef AQ_EVENT_COUNTERS
+ || sc->sc_poll_statistics
+#endif
+ ) {
+ callout_schedule(&sc->sc_tick_ch, hz);
+ }
+
+ /* ready */
+ ifp->if_flags |= IFF_RUNNING;
+ ifp->if_flags &= ~IFF_OACTIVE;
+
+ /* start TX and RX */
+ aq_enable_intr(sc, true, true);
+ AQ_WRITE_REG_BIT(sc, TPB_TX_BUF_REG, TPB_TX_BUF_EN, 1);
+ AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_BUF_EN, 1);
+
+ aq_init_failure:
+ sc->sc_if_flags = ifp->if_flags;
+
+ AQ_UNLOCK(sc);
+
+ return error;
+}
+
+static void
+aq_send_common_locked(struct ifnet *ifp, struct aq_softc *sc,
+ struct aq_txring *txring, bool is_transmit)
+{
+ struct mbuf *m;
+ int npkt, error;
+
+ if ((ifp->if_flags & IFF_RUNNING) == 0)
+ return;
+
+ for (npkt = 0; ; npkt++) {
+ if (is_transmit)
+ m = pcq_peek(txring->txr_pcq);
+ else
+ IFQ_POLL(&ifp->if_snd, m);
+
+ if (m == NULL)
+ break;
+
+ if (txring->txr_nfree < AQ_TXD_MIN)
+ break;
+
+ if (is_transmit)
+ pcq_get(txring->txr_pcq);
+ else
+ IFQ_DEQUEUE(&ifp->if_snd, m);
+
+ error = aq_encap_txring(sc, txring, &m);
+ if (error != 0) {
+ /* too many mbuf chains? or not enough descriptors? */
+ m_freem(m);
+ ifp->if_oerrors++;
+ if (error == ENOBUFS)
+ ifp->if_flags |= IFF_OACTIVE;
+ break;
+ }
+
+ /* update tail ptr */
+ AQ_WRITE_REG(sc, TX_DMA_DESC_TAIL_PTR_REG(txring->txr_index),
+ txring->txr_prodidx);
+
+ /* Pass the packet to any BPF listeners */
+ bpf_mtap(ifp, m, BPF_D_OUT);
+ }
+
+ if (!is_transmit && txring->txr_nfree < AQ_TXD_MIN)
+ ifp->if_flags |= IFF_OACTIVE;
+
+ if (npkt)
+ ifp->if_timer = 5;
+}
+
+static void
+aq_start(struct ifnet *ifp)
+{
+ struct aq_softc *sc;
+ struct aq_txring *txring;
+
+ sc = ifp->if_softc;
+ txring = &sc->sc_queue[0].txring; /* aq_start() always use TX ring[0] */
+
+ mutex_enter(&txring->txr_mutex);
+ if (txring->txr_active && !ISSET(ifp->if_flags, IFF_OACTIVE))
+ aq_send_common_locked(ifp, sc, txring, false);
+ mutex_exit(&txring->txr_mutex);
+}
+
+static inline unsigned int
+aq_select_txqueue(struct aq_softc *sc, struct mbuf *m)
+{
+ return (cpu_index(curcpu()) % sc->sc_nqueues);
+}
+
+static int
+aq_transmit(struct ifnet *ifp, struct mbuf *m)
+{
+ struct aq_softc *sc = ifp->if_softc;
+ struct aq_txring *txring;
+ int ringidx;
+
+ ringidx = aq_select_txqueue(sc, m);
+ txring = &sc->sc_queue[ringidx].txring;
+
+ if (__predict_false(!pcq_put(txring->txr_pcq, m))) {
+ m_freem(m);
+ return ENOBUFS;
+ }
+
+ if (mutex_tryenter(&txring->txr_mutex)) {
+ aq_send_common_locked(ifp, sc, txring, true);
+ mutex_exit(&txring->txr_mutex);
+ } else {
+ softint_schedule(txring->txr_softint);
+ }
+ return 0;
+}
+
+static void
+aq_deferred_transmit(void *arg)
+{
+ struct aq_txring *txring = arg;
+ struct aq_softc *sc = txring->txr_sc;
+ struct ifnet *ifp = &sc->sc_ethercom.ec_if;
+
+ mutex_enter(&txring->txr_mutex);
+ if (pcq_peek(txring->txr_pcq) != NULL)
+ aq_send_common_locked(ifp, sc, txring, true);
+ mutex_exit(&txring->txr_mutex);
+}
+
+static void
+aq_stop(struct ifnet *ifp, int disable)
+{
+ struct aq_softc *sc = ifp->if_softc;
+ int i;
+
+ AQ_LOCK(sc);
+
+ ifp->if_timer = 0;
+
+ /* disable tx/rx interrupts */
+ aq_enable_intr(sc, true, false);
+
+ AQ_WRITE_REG_BIT(sc, TPB_TX_BUF_REG, TPB_TX_BUF_EN, 0);
+ for (i = 0; i < sc->sc_nqueues; i++) {
+ aq_txring_reset(sc, &sc->sc_queue[i].txring, false);
+ }
+
+ AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_BUF_EN, 0);
+ for (i = 0; i < sc->sc_nqueues; i++) {
+ aq_rxring_reset(sc, &sc->sc_queue[i].rxring, false);
+ }
+
+ /* invalidate RX descriptor cache */
+ AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_CACHE_INIT_REG, RX_DMA_DESC_CACHE_INIT,
+ AQ_READ_REG_BIT(sc,
+ RX_DMA_DESC_CACHE_INIT_REG, RX_DMA_DESC_CACHE_INIT) ^ 1);
+
+ ifp->if_timer = 0;
+
+ if (!disable) {
+ /* when pmf stop, disable link status intr and callout */
+ aq_enable_intr(sc, false, false);
+ callout_stop(&sc->sc_tick_ch);
+ }
+
+ ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
+
+ AQ_UNLOCK(sc);
+}
+
+static void
+aq_watchdog(struct ifnet *ifp)
+{
+ struct aq_softc *sc = ifp->if_softc;
+ struct aq_txring *txring;
+ int n, head, tail;
+
+ AQ_LOCK(sc);
+
+ device_printf(sc->sc_dev, "%s: INTR_MASK/STATUS = %08x/%08x\n",
+ __func__, AQ_READ_REG(sc, AQ_INTR_MASK_REG),
+ AQ_READ_REG(sc, AQ_INTR_STATUS_REG));
+
+ for (n = 0; n < sc->sc_nqueues; n++) {
+ txring = &sc->sc_queue[n].txring;
+ head = AQ_READ_REG_BIT(sc,
+ TX_DMA_DESC_HEAD_PTR_REG(txring->txr_index),
+ TX_DMA_DESC_HEAD_PTR),
+ tail = AQ_READ_REG(sc,
+ TX_DMA_DESC_TAIL_PTR_REG(txring->txr_index));
+
+ device_printf(sc->sc_dev, "%s: TXring[%d] HEAD/TAIL=%d/%d\n",
+ __func__, txring->txr_index, head, tail);
+
+ aq_tx_intr(txring);
+ }
+
+ AQ_UNLOCK(sc);
+
+ aq_init(ifp);
+}
+
+static int
+aq_ioctl(struct ifnet *ifp, unsigned long cmd, void *data)
+{
+ struct aq_softc *sc __unused;
+ struct ifreq *ifr __unused;
+ int error, s;
+
+ sc = (struct aq_softc *)ifp->if_softc;
+ ifr = (struct ifreq *)data;
+ error = 0;
+
+ s = splnet();
+ error = ether_ioctl(ifp, cmd, data);
+ splx(s);
+
+ if (error != ENETRESET)
+ return error;
+
+ switch (cmd) {
+ case SIOCSIFCAP:
+ error = aq_set_capability(sc);
+ break;
+ case SIOCADDMULTI:
+ case SIOCDELMULTI:
+ if ((ifp->if_flags & IFF_RUNNING) == 0)
+ break;
+
+ /*
+ * Multicast list has changed; set the hardware filter
+ * accordingly.
+ */
+ error = aq_set_filter(sc);
+ break;
+ }
+
+ return error;
+}
+
+
+MODULE(MODULE_CLASS_DRIVER, if_aq, "pci");
+
+#ifdef _MODULE
+#include "ioconf.c"
+#endif
+
+static int
+if_aq_modcmd(modcmd_t cmd, void *opaque)
+{
+ int error = 0;
+
+ switch (cmd) {
+ case MODULE_CMD_INIT:
+#ifdef _MODULE
+ error = config_init_component(cfdriver_ioconf_if_aq,
+ cfattach_ioconf_if_aq, cfdata_ioconf_if_aq);
+#endif
+ return error;
+ case MODULE_CMD_FINI:
+#ifdef _MODULE
+ error = config_fini_component(cfdriver_ioconf_if_aq,
+ cfattach_ioconf_if_aq, cfdata_ioconf_if_aq);
+#endif
+ return error;
+ default:
+ return ENOTTY;
+ }
+}
