diff options
Diffstat (limited to 'freebsd/sys/dev/e1000/if_em.c')
| -rw-r--r-- | freebsd/sys/dev/e1000/if_em.c | 5878 |
1 files changed, 2005 insertions, 3873 deletions
diff --git a/freebsd/sys/dev/e1000/if_em.c b/freebsd/sys/dev/e1000/if_em.c index d8c1e5a6..2054f994 100644 --- a/freebsd/sys/dev/e1000/if_em.c +++ b/freebsd/sys/dev/e1000/if_em.c @@ -1,101 +1,38 @@ #include <machine/rtems-bsd-kernel-space.h> -/****************************************************************************** - - Copyright (c) 2001-2015, Intel Corporation - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - -******************************************************************************/ -/*$FreeBSD$*/ - -#include <rtems/bsd/local/opt_em.h> -#include <rtems/bsd/local/opt_ddb.h> -#include <rtems/bsd/local/opt_inet.h> -#include <rtems/bsd/local/opt_inet6.h> - -#ifdef HAVE_KERNEL_OPTION_HEADERS -#include <rtems/bsd/local/opt_device_polling.h> -#endif +/*- + * Copyright (c) 2016 Matt Macy <mmacy@nextbsd.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ -#include <rtems/bsd/sys/param.h> -#include <sys/systm.h> -#ifdef DDB -#include <sys/types.h> -#include <ddb/ddb.h> -#endif -#if __FreeBSD_version >= 800000 -#include <sys/buf_ring.h> -#endif -#include <sys/bus.h> -#include <sys/endian.h> -#include <sys/kernel.h> -#include <sys/kthread.h> -#include <sys/malloc.h> -#include <sys/mbuf.h> -#include <sys/module.h> -#include <sys/rman.h> -#include <sys/smp.h> -#include <sys/socket.h> -#include <sys/sockio.h> -#include <sys/sysctl.h> -#include <sys/taskqueue.h> -#include <sys/eventhandler.h> -#include <machine/bus.h> -#include <machine/resource.h> - -#include <net/bpf.h> -#include <net/ethernet.h> -#include <net/if.h> -#include <net/if_var.h> -#include <net/if_arp.h> -#include <net/if_dl.h> -#include <net/if_media.h> - -#include <net/if_types.h> -#include <net/if_vlan_var.h> - -#include <netinet/in_systm.h> -#include <netinet/in.h> -#include <netinet/if_ether.h> -#include <netinet/ip.h> -#include <netinet/ip6.h> -#include <netinet/tcp.h> -#include <netinet/udp.h> - -#include <machine/in_cksum.h> -#include <dev/led/led.h> -#include <dev/pci/pcivar.h> -#include <dev/pci/pcireg.h> - -#include "e1000_api.h" -#include "e1000_82571.h" +/* $FreeBSD$ */ #include "if_em.h" +#include <sys/sbuf.h> +#include <machine/_inttypes.h> + +#define em_mac_min e1000_82547 +#define igb_mac_min e1000_82575 /********************************************************************* * Driver version: @@ -112,187 +49,224 @@ char em_driver_version[] = "7.6.1-k"; * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } *********************************************************************/ -static em_vendor_info_t em_vendor_info_array[] = +static pci_vendor_info_t em_vendor_info_array[] = { - /* Intel(R) PRO/1000 Network Connection */ - { 0x8086, E1000_DEV_ID_82571EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82571EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82571EB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82572EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82572EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82572EI_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82572EI, PCI_ANY_ID, PCI_ANY_ID, 0}, - - { 0x8086, E1000_DEV_ID_82573E, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82573E_IAMT, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82573L, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82583V, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH8_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH8_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH8_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH8_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH8_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH8_82567V_3, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_BM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82574L, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82574LA, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH10_R_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH10_D_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_M_HV_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_M_HV_LC, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_D_HV_DM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_D_HV_DC, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH2_LV_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH2_LV_V, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_LPT_I217_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_LPT_I217_V, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_LPTLP_I218_LM, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_LPTLP_I218_V, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_I218_LM2, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_I218_V2, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_I218_LM3, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_I218_V3, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM2, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V2, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_LBG_I219_LM3, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM4, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V4, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM5, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V5, PCI_ANY_ID, PCI_ANY_ID, 0}, + /* Intel(R) PRO/1000 Network Connection - Legacy em*/ + PVID(0x8086, E1000_DEV_ID_82540EM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82540EM_LOM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82540EP, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82540EP_LOM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82540EP_LP, "Intel(R) PRO/1000 Network Connection"), + + PVID(0x8086, E1000_DEV_ID_82541EI, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82541ER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82541ER_LOM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82541EI_MOBILE, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82541GI, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82541GI_LF, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82541GI_MOBILE, "Intel(R) PRO/1000 Network Connection"), + + PVID(0x8086, E1000_DEV_ID_82542, "Intel(R) PRO/1000 Network Connection"), + + PVID(0x8086, E1000_DEV_ID_82543GC_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82543GC_COPPER, "Intel(R) PRO/1000 Network Connection"), + + PVID(0x8086, E1000_DEV_ID_82544EI_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82544EI_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82544GC_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82544GC_LOM, "Intel(R) PRO/1000 Network Connection"), + + PVID(0x8086, E1000_DEV_ID_82545EM_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82545EM_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82545GM_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82545GM_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82545GM_SERDES, "Intel(R) PRO/1000 Network Connection"), + + PVID(0x8086, E1000_DEV_ID_82546EB_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82546EB_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82546GB_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82546GB_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82546GB_SERDES, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82546GB_PCIE, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3, "Intel(R) PRO/1000 Network Connection"), + + PVID(0x8086, E1000_DEV_ID_82547EI, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82547EI_MOBILE, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82547GI, "Intel(R) PRO/1000 Network Connection"), + + /* Intel(R) PRO/1000 Network Connection - em */ + PVID(0x8086, E1000_DEV_ID_82571EB_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82571EB_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82571EB_SERDES, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82572EI, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82572EI_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82572EI_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82572EI_SERDES, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82573E, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82573E_IAMT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82573L, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82583V, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH8_IGP_AMT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH8_IGP_C, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH8_IFE, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH8_IFE_GT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH8_IFE_G, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH8_IGP_M, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH8_82567V_3, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_IGP_AMT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_IGP_C, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M_V, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_IFE, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_IFE_GT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_IFE_G, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_BM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82574L, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82574LA, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_LM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_LF, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_V, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_LM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_LF, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_V, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_M_HV_LM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_M_HV_LC, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_D_HV_DM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_D_HV_DC, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH2_LV_LM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH2_LV_V, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_LPT_I217_LM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_LPT_I217_V, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_LPTLP_I218_LM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_LPTLP_I218_V, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_I218_LM2, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_I218_V2, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_I218_LM3, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_I218_V3, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM2, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V2, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_LBG_I219_LM3, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM4, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V4, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM5, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V5, "Intel(R) PRO/1000 Network Connection"), /* required last entry */ - { 0, 0, 0, 0, 0} + PVID_END }; -/********************************************************************* - * Table of branding strings for all supported NICs. - *********************************************************************/ - -static char *em_strings[] = { - "Intel(R) PRO/1000 Network Connection" +static pci_vendor_info_t igb_vendor_info_array[] = +{ + /* Intel(R) PRO/1000 Network Connection - igb */ + PVID(0x8086, E1000_DEV_ID_82575EB_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576_NS, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576_NS_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576_SERDES_QUAD, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576_QUAD_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576_QUAD_COPPER_ET2, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576_VF, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82580_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82580_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82580_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82580_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82580_COPPER_DUAL, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82580_QUAD_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_DH89XXCC_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_DH89XXCC_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_DH89XXCC_SFP, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_DH89XXCC_BACKPLANE, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I350_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I350_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I350_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I350_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I350_VF, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I210_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I210_COPPER_IT, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I210_COPPER_OEM1, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I210_COPPER_FLASHLESS, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I210_SERDES_FLASHLESS, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I210_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I210_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I210_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I211_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I354_BACKPLANE_1GBPS, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I354_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"), + /* required last entry */ + PVID_END }; /********************************************************************* * Function prototypes *********************************************************************/ -static int em_probe(device_t); -static int em_attach(device_t); -static int em_detach(device_t); -static int em_shutdown(device_t); -static int em_suspend(device_t); -static int em_resume(device_t); -#ifdef EM_MULTIQUEUE -static int em_mq_start(if_t, struct mbuf *); -static int em_mq_start_locked(if_t, - struct tx_ring *); -static void em_qflush(if_t); -#else -static void em_start(if_t); -static void em_start_locked(if_t, struct tx_ring *); -#endif -static int em_ioctl(if_t, u_long, caddr_t); -static uint64_t em_get_counter(if_t, ift_counter); -static void em_init(void *); -static void em_init_locked(struct adapter *); -static void em_stop(void *); -static void em_media_status(if_t, struct ifmediareq *); -static int em_media_change(if_t); -static void em_identify_hardware(struct adapter *); -static int em_allocate_pci_resources(struct adapter *); -static int em_allocate_legacy(struct adapter *); -static int em_allocate_msix(struct adapter *); -static int em_allocate_queues(struct adapter *); -static int em_setup_msix(struct adapter *); -static void em_free_pci_resources(struct adapter *); -static void em_local_timer(void *); -static void em_reset(struct adapter *); -static int em_setup_interface(device_t, struct adapter *); -static void em_flush_desc_rings(struct adapter *); - -static void em_setup_transmit_structures(struct adapter *); -static void em_initialize_transmit_unit(struct adapter *); -static int em_allocate_transmit_buffers(struct tx_ring *); -static void em_free_transmit_structures(struct adapter *); -static void em_free_transmit_buffers(struct tx_ring *); - -static int em_setup_receive_structures(struct adapter *); -static int em_allocate_receive_buffers(struct rx_ring *); -static void em_initialize_receive_unit(struct adapter *); -static void em_free_receive_structures(struct adapter *); -static void em_free_receive_buffers(struct rx_ring *); - -static void em_enable_intr(struct adapter *); -static void em_disable_intr(struct adapter *); +static void *em_register(device_t dev); +static void *igb_register(device_t dev); +static int em_if_attach_pre(if_ctx_t ctx); +static int em_if_attach_post(if_ctx_t ctx); +static int em_if_detach(if_ctx_t ctx); +static int em_if_shutdown(if_ctx_t ctx); +static int em_if_suspend(if_ctx_t ctx); +static int em_if_resume(if_ctx_t ctx); + +static int em_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets); +static int em_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nrxqs, int nrxqsets); +static void em_if_queues_free(if_ctx_t ctx); + +static uint64_t em_if_get_counter(if_ctx_t, ift_counter); +static void em_if_init(if_ctx_t ctx); +static void em_if_stop(if_ctx_t ctx); +static void em_if_media_status(if_ctx_t, struct ifmediareq *); +static int em_if_media_change(if_ctx_t ctx); +static int em_if_mtu_set(if_ctx_t ctx, uint32_t mtu); +static void em_if_timer(if_ctx_t ctx, uint16_t qid); +static void em_if_vlan_register(if_ctx_t ctx, u16 vtag); +static void em_if_vlan_unregister(if_ctx_t ctx, u16 vtag); + +static void em_identify_hardware(if_ctx_t ctx); +static int em_allocate_pci_resources(if_ctx_t ctx); +static void em_free_pci_resources(if_ctx_t ctx); +static void em_reset(if_ctx_t ctx); +static int em_setup_interface(if_ctx_t ctx); +static int em_setup_msix(if_ctx_t ctx); + +static void em_initialize_transmit_unit(if_ctx_t ctx); +static void em_initialize_receive_unit(if_ctx_t ctx); + +static void em_if_enable_intr(if_ctx_t ctx); +static void em_if_disable_intr(if_ctx_t ctx); +static int em_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid); +static int em_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid); +static void em_if_multi_set(if_ctx_t ctx); +static void em_if_update_admin_status(if_ctx_t ctx); +static void em_if_debug(if_ctx_t ctx); static void em_update_stats_counters(struct adapter *); static void em_add_hw_stats(struct adapter *adapter); -static void em_txeof(struct tx_ring *); -static bool em_rxeof(struct rx_ring *, int, int *); -#ifndef __NO_STRICT_ALIGNMENT -static int em_fixup_rx(struct rx_ring *); -#endif -static void em_setup_rxdesc(union e1000_rx_desc_extended *, - const struct em_rxbuffer *rxbuf); -static void em_receive_checksum(uint32_t status, struct mbuf *); -static void em_transmit_checksum_setup(struct tx_ring *, struct mbuf *, int, - struct ip *, u32 *, u32 *); -static void em_tso_setup(struct tx_ring *, struct mbuf *, int, struct ip *, - struct tcphdr *, u32 *, u32 *); -static void em_set_promisc(struct adapter *); -static void em_disable_promisc(struct adapter *); -static void em_set_multi(struct adapter *); -static void em_update_link_status(struct adapter *); -static void em_refresh_mbufs(struct rx_ring *, int); -static void em_register_vlan(void *, if_t, u16); -static void em_unregister_vlan(void *, if_t, u16); +static int em_if_set_promisc(if_ctx_t ctx, int flags); static void em_setup_vlan_hw_support(struct adapter *); -static int em_xmit(struct tx_ring *, struct mbuf **); -static int em_dma_malloc(struct adapter *, bus_size_t, - struct em_dma_alloc *, int); -static void em_dma_free(struct adapter *, struct em_dma_alloc *); static int em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS); static void em_print_nvm_info(struct adapter *); static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS); +static int em_get_rs(SYSCTL_HANDLER_ARGS); static void em_print_debug_info(struct adapter *); static int em_is_valid_ether_addr(u8 *); static int em_sysctl_int_delay(SYSCTL_HANDLER_ARGS); @@ -301,65 +275,123 @@ static void em_add_int_delay_sysctl(struct adapter *, const char *, /* Management and WOL Support */ static void em_init_manageability(struct adapter *); static void em_release_manageability(struct adapter *); -static void em_get_hw_control(struct adapter *); -static void em_release_hw_control(struct adapter *); -static void em_get_wakeup(device_t); -static void em_enable_wakeup(device_t); +static void em_get_hw_control(struct adapter *); +static void em_release_hw_control(struct adapter *); +static void em_get_wakeup(if_ctx_t ctx); +static void em_enable_wakeup(if_ctx_t ctx); static int em_enable_phy_wakeup(struct adapter *); -static void em_led_func(void *, int); static void em_disable_aspm(struct adapter *); -static int em_irq_fast(void *); +int em_intr(void *arg); +static void em_disable_promisc(if_ctx_t ctx); /* MSIX handlers */ -static void em_msix_tx(void *); -static void em_msix_rx(void *); -static void em_msix_link(void *); -static void em_handle_tx(void *context, int pending); -static void em_handle_rx(void *context, int pending); -static void em_handle_link(void *context, int pending); - -#ifdef EM_MULTIQUEUE -static void em_enable_vectors_82574(struct adapter *); -#endif +static int em_if_msix_intr_assign(if_ctx_t, int); +static int em_msix_link(void *); +static void em_handle_link(void *context); + +static void em_enable_vectors_82574(if_ctx_t); -static void em_set_sysctl_value(struct adapter *, const char *, - const char *, int *, int); static int em_set_flowcntl(SYSCTL_HANDLER_ARGS); static int em_sysctl_eee(SYSCTL_HANDLER_ARGS); +static void em_if_led_func(if_ctx_t ctx, int onoff); -static __inline void em_rx_discard(struct rx_ring *, int); +static int em_get_regs(SYSCTL_HANDLER_ARGS); + +static void lem_smartspeed(struct adapter *adapter); +static void igb_configure_queues(struct adapter *adapter); -#ifdef DEVICE_POLLING -static poll_handler_t em_poll; -#endif /* POLLING */ /********************************************************************* * FreeBSD Device Interface Entry Points *********************************************************************/ - static device_method_t em_methods[] = { /* Device interface */ - DEVMETHOD(device_probe, em_probe), - DEVMETHOD(device_attach, em_attach), - DEVMETHOD(device_detach, em_detach), - DEVMETHOD(device_shutdown, em_shutdown), - DEVMETHOD(device_suspend, em_suspend), - DEVMETHOD(device_resume, em_resume), + DEVMETHOD(device_register, em_register), + DEVMETHOD(device_probe, iflib_device_probe), + DEVMETHOD(device_attach, iflib_device_attach), + DEVMETHOD(device_detach, iflib_device_detach), + DEVMETHOD(device_shutdown, iflib_device_shutdown), + DEVMETHOD(device_suspend, iflib_device_suspend), + DEVMETHOD(device_resume, iflib_device_resume), + DEVMETHOD_END +}; + +static device_method_t igb_methods[] = { + /* Device interface */ + DEVMETHOD(device_register, igb_register), + DEVMETHOD(device_probe, iflib_device_probe), + DEVMETHOD(device_attach, iflib_device_attach), + DEVMETHOD(device_detach, iflib_device_detach), + DEVMETHOD(device_shutdown, iflib_device_shutdown), + DEVMETHOD(device_suspend, iflib_device_suspend), + DEVMETHOD(device_resume, iflib_device_resume), DEVMETHOD_END }; + static driver_t em_driver = { "em", em_methods, sizeof(struct adapter), }; -devclass_t em_devclass; +static devclass_t em_devclass; DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0); + MODULE_DEPEND(em, pci, 1, 1, 1); MODULE_DEPEND(em, ether, 1, 1, 1); -#ifdef DEV_NETMAP -MODULE_DEPEND(em, netmap, 1, 1, 1); -#endif /* DEV_NETMAP */ +MODULE_DEPEND(em, iflib, 1, 1, 1); + +static driver_t igb_driver = { + "igb", igb_methods, sizeof(struct adapter), +}; + +static devclass_t igb_devclass; +DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0); + +MODULE_DEPEND(igb, pci, 1, 1, 1); +MODULE_DEPEND(igb, ether, 1, 1, 1); +MODULE_DEPEND(igb, iflib, 1, 1, 1); + + +static device_method_t em_if_methods[] = { + DEVMETHOD(ifdi_attach_pre, em_if_attach_pre), + DEVMETHOD(ifdi_attach_post, em_if_attach_post), + DEVMETHOD(ifdi_detach, em_if_detach), + DEVMETHOD(ifdi_shutdown, em_if_shutdown), + DEVMETHOD(ifdi_suspend, em_if_suspend), + DEVMETHOD(ifdi_resume, em_if_resume), + DEVMETHOD(ifdi_init, em_if_init), + DEVMETHOD(ifdi_stop, em_if_stop), + DEVMETHOD(ifdi_msix_intr_assign, em_if_msix_intr_assign), + DEVMETHOD(ifdi_intr_enable, em_if_enable_intr), + DEVMETHOD(ifdi_intr_disable, em_if_disable_intr), + DEVMETHOD(ifdi_tx_queues_alloc, em_if_tx_queues_alloc), + DEVMETHOD(ifdi_rx_queues_alloc, em_if_rx_queues_alloc), + DEVMETHOD(ifdi_queues_free, em_if_queues_free), + DEVMETHOD(ifdi_update_admin_status, em_if_update_admin_status), + DEVMETHOD(ifdi_multi_set, em_if_multi_set), + DEVMETHOD(ifdi_media_status, em_if_media_status), + DEVMETHOD(ifdi_media_change, em_if_media_change), + DEVMETHOD(ifdi_mtu_set, em_if_mtu_set), + DEVMETHOD(ifdi_promisc_set, em_if_set_promisc), + DEVMETHOD(ifdi_timer, em_if_timer), + DEVMETHOD(ifdi_vlan_register, em_if_vlan_register), + DEVMETHOD(ifdi_vlan_unregister, em_if_vlan_unregister), + DEVMETHOD(ifdi_get_counter, em_if_get_counter), + DEVMETHOD(ifdi_led_func, em_if_led_func), + DEVMETHOD(ifdi_rx_queue_intr_enable, em_if_rx_queue_intr_enable), + DEVMETHOD(ifdi_tx_queue_intr_enable, em_if_tx_queue_intr_enable), + DEVMETHOD(ifdi_debug, em_if_debug), + DEVMETHOD_END +}; + +/* + * note that if (adapter->msix_mem) is replaced by: + * if (adapter->intr_type == IFLIB_INTR_MSIX) + */ +static driver_t em_if_driver = { + "em_if", em_if_methods, sizeof(struct adapter) +}; /********************************************************************* * Tunable default values. @@ -367,10 +399,16 @@ MODULE_DEPEND(em, netmap, 1, 1, 1); #define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000) #define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024) +#define M_TSO_LEN 66 #define MAX_INTS_PER_SEC 8000 #define DEFAULT_ITR (1000000000/(MAX_INTS_PER_SEC * 256)) +/* Allow common code without TSO */ +#ifndef CSUM_TSO +#define CSUM_TSO 0 +#endif + #define TSO_WORKAROUND 4 static SYSCTL_NODE(_hw, OID_AUTO, em, CTLFLAG_RD, 0, "EM driver parameters"); @@ -395,39 +433,15 @@ SYSCTL_INT(_hw_em, OID_AUTO, rx_abs_int_delay, CTLFLAG_RDTUN, &em_rx_abs_int_delay_dflt, 0, "Default receive interrupt delay limit in usecs"); -static int em_rxd = EM_DEFAULT_RXD; -static int em_txd = EM_DEFAULT_TXD; -SYSCTL_INT(_hw_em, OID_AUTO, rxd, CTLFLAG_RDTUN, &em_rxd, 0, - "Number of receive descriptors per queue"); -SYSCTL_INT(_hw_em, OID_AUTO, txd, CTLFLAG_RDTUN, &em_txd, 0, - "Number of transmit descriptors per queue"); - static int em_smart_pwr_down = FALSE; SYSCTL_INT(_hw_em, OID_AUTO, smart_pwr_down, CTLFLAG_RDTUN, &em_smart_pwr_down, 0, "Set to true to leave smart power down enabled on newer adapters"); /* Controls whether promiscuous also shows bad packets */ -static int em_debug_sbp = FALSE; +static int em_debug_sbp = TRUE; SYSCTL_INT(_hw_em, OID_AUTO, sbp, CTLFLAG_RDTUN, &em_debug_sbp, 0, "Show bad packets in promiscuous mode"); -static int em_enable_msix = TRUE; -SYSCTL_INT(_hw_em, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &em_enable_msix, 0, - "Enable MSI-X interrupts"); - -#ifdef EM_MULTIQUEUE -static int em_num_queues = 1; -SYSCTL_INT(_hw_em, OID_AUTO, num_queues, CTLFLAG_RDTUN, &em_num_queues, 0, - "82574 only: Number of queues to configure, 0 indicates autoconfigure"); -#endif - -/* -** Global variable to store last used CPU when binding queues -** to CPUs in igb_allocate_msix. Starts at CPU_FIRST and increments when a -** queue is bound to a cpu. -*/ -static int em_last_bind_cpu = -1; - /* How many packets rxeof tries to clean at a time */ static int em_rx_process_limit = 100; SYSCTL_INT(_hw_em, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, @@ -440,64 +454,243 @@ static int eee_setting = 1; SYSCTL_INT(_hw_em, OID_AUTO, eee_setting, CTLFLAG_RDTUN, &eee_setting, 0, "Enable Energy Efficient Ethernet"); +/* +** Tuneable Interrupt rate +*/ +static int em_max_interrupt_rate = 8000; +SYSCTL_INT(_hw_em, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN, + &em_max_interrupt_rate, 0, "Maximum interrupts per second"); + + + /* Global used in WOL setup with multiport cards */ static int global_quad_port_a = 0; -#ifdef DEV_NETMAP /* see ixgbe.c for details */ -#include <dev/netmap/if_em_netmap.h> -#endif /* DEV_NETMAP */ +extern struct if_txrx igb_txrx; +extern struct if_txrx em_txrx; +extern struct if_txrx lem_txrx; + +static struct if_shared_ctx em_sctx_init = { + .isc_magic = IFLIB_MAGIC, + .isc_q_align = PAGE_SIZE, + .isc_tx_maxsize = EM_TSO_SIZE, + .isc_tx_maxsegsize = PAGE_SIZE, + .isc_rx_maxsize = MJUM9BYTES, + .isc_rx_nsegments = 1, + .isc_rx_maxsegsize = MJUM9BYTES, + .isc_nfl = 1, + .isc_nrxqs = 1, + .isc_ntxqs = 1, + .isc_admin_intrcnt = 1, + .isc_vendor_info = em_vendor_info_array, + .isc_driver_version = em_driver_version, + .isc_driver = &em_if_driver, + .isc_flags = IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP, + + .isc_nrxd_min = {EM_MIN_RXD}, + .isc_ntxd_min = {EM_MIN_TXD}, + .isc_nrxd_max = {EM_MAX_RXD}, + .isc_ntxd_max = {EM_MAX_TXD}, + .isc_nrxd_default = {EM_DEFAULT_RXD}, + .isc_ntxd_default = {EM_DEFAULT_TXD}, +}; -/********************************************************************* - * Device identification routine +if_shared_ctx_t em_sctx = &em_sctx_init; + + +static struct if_shared_ctx igb_sctx_init = { + .isc_magic = IFLIB_MAGIC, + .isc_q_align = PAGE_SIZE, + .isc_tx_maxsize = EM_TSO_SIZE, + .isc_tx_maxsegsize = PAGE_SIZE, + .isc_rx_maxsize = MJUM9BYTES, + .isc_rx_nsegments = 1, + .isc_rx_maxsegsize = MJUM9BYTES, + .isc_nfl = 1, + .isc_nrxqs = 1, + .isc_ntxqs = 1, + .isc_admin_intrcnt = 1, + .isc_vendor_info = igb_vendor_info_array, + .isc_driver_version = em_driver_version, + .isc_driver = &em_if_driver, + .isc_flags = IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP, + + .isc_nrxd_min = {EM_MIN_RXD}, + .isc_ntxd_min = {EM_MIN_TXD}, + .isc_nrxd_max = {EM_MAX_RXD}, + .isc_ntxd_max = {EM_MAX_TXD}, + .isc_nrxd_default = {EM_DEFAULT_RXD}, + .isc_ntxd_default = {EM_DEFAULT_TXD}, +}; + +if_shared_ctx_t igb_sctx = &igb_sctx_init; + +/***************************************************************** * - * em_probe determines if the driver should be loaded on - * adapter based on PCI vendor/device id of the adapter. + * Dump Registers * - * return BUS_PROBE_DEFAULT on success, positive on failure - *********************************************************************/ + ****************************************************************/ +#define IGB_REGS_LEN 739 -static int -em_probe(device_t dev) +static int em_get_regs(SYSCTL_HANDLER_ARGS) { - char adapter_name[60]; - uint16_t pci_vendor_id = 0; - uint16_t pci_device_id = 0; - uint16_t pci_subvendor_id = 0; - uint16_t pci_subdevice_id = 0; - em_vendor_info_t *ent; + struct adapter *adapter = (struct adapter *)arg1; + struct e1000_hw *hw = &adapter->hw; - INIT_DEBUGOUT("em_probe: begin"); + struct sbuf *sb; + u32 *regs_buff = (u32 *)malloc(sizeof(u32) * IGB_REGS_LEN, M_DEVBUF, M_NOWAIT); + int rc; - pci_vendor_id = pci_get_vendor(dev); - if (pci_vendor_id != EM_VENDOR_ID) - return (ENXIO); + memset(regs_buff, 0, IGB_REGS_LEN * sizeof(u32)); - pci_device_id = pci_get_device(dev); - pci_subvendor_id = pci_get_subvendor(dev); - pci_subdevice_id = pci_get_subdevice(dev); - - ent = em_vendor_info_array; - while (ent->vendor_id != 0) { - if ((pci_vendor_id == ent->vendor_id) && - (pci_device_id == ent->device_id) && - - ((pci_subvendor_id == ent->subvendor_id) || - (ent->subvendor_id == PCI_ANY_ID)) && - - ((pci_subdevice_id == ent->subdevice_id) || - (ent->subdevice_id == PCI_ANY_ID))) { - sprintf(adapter_name, "%s %s", - em_strings[ent->index], - em_driver_version); - device_set_desc_copy(dev, adapter_name); - return (BUS_PROBE_DEFAULT); - } - ent++; + rc = sysctl_wire_old_buffer(req, 0); + MPASS(rc == 0); + if (rc != 0) + return (rc); + + sb = sbuf_new_for_sysctl(NULL, NULL, 32*400, req); + MPASS(sb != NULL); + if (sb == NULL) + return (ENOMEM); + + /* General Registers */ + regs_buff[0] = E1000_READ_REG(hw, E1000_CTRL); + regs_buff[1] = E1000_READ_REG(hw, E1000_STATUS); + regs_buff[2] = E1000_READ_REG(hw, E1000_CTRL_EXT); + regs_buff[3] = E1000_READ_REG(hw, E1000_ICR); + regs_buff[4] = E1000_READ_REG(hw, E1000_RCTL); + regs_buff[5] = E1000_READ_REG(hw, E1000_RDLEN(0)); + regs_buff[6] = E1000_READ_REG(hw, E1000_RDH(0)); + regs_buff[7] = E1000_READ_REG(hw, E1000_RDT(0)); + regs_buff[8] = E1000_READ_REG(hw, E1000_RXDCTL(0)); + regs_buff[9] = E1000_READ_REG(hw, E1000_RDBAL(0)); + regs_buff[10] = E1000_READ_REG(hw, E1000_RDBAH(0)); + regs_buff[11] = E1000_READ_REG(hw, E1000_TCTL); + regs_buff[12] = E1000_READ_REG(hw, E1000_TDBAL(0)); + regs_buff[13] = E1000_READ_REG(hw, E1000_TDBAH(0)); + regs_buff[14] = E1000_READ_REG(hw, E1000_TDLEN(0)); + regs_buff[15] = E1000_READ_REG(hw, E1000_TDH(0)); + regs_buff[16] = E1000_READ_REG(hw, E1000_TDT(0)); + regs_buff[17] = E1000_READ_REG(hw, E1000_TXDCTL(0)); + regs_buff[18] = E1000_READ_REG(hw, E1000_TDFH); + regs_buff[19] = E1000_READ_REG(hw, E1000_TDFT); + regs_buff[20] = E1000_READ_REG(hw, E1000_TDFHS); + regs_buff[21] = E1000_READ_REG(hw, E1000_TDFPC); + + sbuf_printf(sb, "General Registers\n"); + sbuf_printf(sb, "\tCTRL\t %08x\n", regs_buff[0]); + sbuf_printf(sb, "\tSTATUS\t %08x\n", regs_buff[1]); + sbuf_printf(sb, "\tCTRL_EXIT\t %08x\n\n", regs_buff[2]); + + sbuf_printf(sb, "Interrupt Registers\n"); + sbuf_printf(sb, "\tICR\t %08x\n\n", regs_buff[3]); + + sbuf_printf(sb, "RX Registers\n"); + sbuf_printf(sb, "\tRCTL\t %08x\n", regs_buff[4]); + sbuf_printf(sb, "\tRDLEN\t %08x\n", regs_buff[5]); + sbuf_printf(sb, "\tRDH\t %08x\n", regs_buff[6]); + sbuf_printf(sb, "\tRDT\t %08x\n", regs_buff[7]); + sbuf_printf(sb, "\tRXDCTL\t %08x\n", regs_buff[8]); + sbuf_printf(sb, "\tRDBAL\t %08x\n", regs_buff[9]); + sbuf_printf(sb, "\tRDBAH\t %08x\n\n", regs_buff[10]); + + sbuf_printf(sb, "TX Registers\n"); + sbuf_printf(sb, "\tTCTL\t %08x\n", regs_buff[11]); + sbuf_printf(sb, "\tTDBAL\t %08x\n", regs_buff[12]); + sbuf_printf(sb, "\tTDBAH\t %08x\n", regs_buff[13]); + sbuf_printf(sb, "\tTDLEN\t %08x\n", regs_buff[14]); + sbuf_printf(sb, "\tTDH\t %08x\n", regs_buff[15]); + sbuf_printf(sb, "\tTDT\t %08x\n", regs_buff[16]); + sbuf_printf(sb, "\tTXDCTL\t %08x\n", regs_buff[17]); + sbuf_printf(sb, "\tTDFH\t %08x\n", regs_buff[18]); + sbuf_printf(sb, "\tTDFT\t %08x\n", regs_buff[19]); + sbuf_printf(sb, "\tTDFHS\t %08x\n", regs_buff[20]); + sbuf_printf(sb, "\tTDFPC\t %08x\n\n", regs_buff[21]); + +#ifdef DUMP_DESCS + { + if_softc_ctx_t scctx = adapter->shared; + struct rx_ring *rxr = &rx_que->rxr; + struct tx_ring *txr = &tx_que->txr; + int ntxd = scctx->isc_ntxd[0]; + int nrxd = scctx->isc_nrxd[0]; + int j; + + for (j = 0; j < nrxd; j++) { + u32 staterr = le32toh(rxr->rx_base[j].wb.upper.status_error); + u32 length = le32toh(rxr->rx_base[j].wb.upper.length); + sbuf_printf(sb, "\tReceive Descriptor Address %d: %08" PRIx64 " Error:%d Length:%d\n", j, rxr->rx_base[j].read.buffer_addr, staterr, length); + } + + for (j = 0; j < min(ntxd, 256); j++) { + unsigned int *ptr = (unsigned int *)&txr->tx_base[j]; + + sbuf_printf(sb, "\tTXD[%03d] [0]: %08x [1]: %08x [2]: %08x [3]: %08x eop: %d DD=%d\n", + j, ptr[0], ptr[1], ptr[2], ptr[3], buf->eop, + buf->eop != -1 ? txr->tx_base[buf->eop].upper.fields.status & E1000_TXD_STAT_DD : 0); + + } } +#endif - return (ENXIO); + rc = sbuf_finish(sb); + sbuf_delete(sb); + return(rc); } +static void * +em_register(device_t dev) +{ + return (em_sctx); +} + +static void * +igb_register(device_t dev) +{ + return (igb_sctx); +} + +static int +em_set_num_queues(if_ctx_t ctx) +{ + struct adapter *adapter = iflib_get_softc(ctx); + int maxqueues; + + /* Sanity check based on HW */ + switch (adapter->hw.mac.type) { + case e1000_82576: + case e1000_82580: + case e1000_i350: + case e1000_i354: + maxqueues = 8; + break; + case e1000_i210: + case e1000_82575: + maxqueues = 4; + break; + case e1000_i211: + case e1000_82574: + maxqueues = 2; + break; + default: + maxqueues = 1; + break; + } + + return (maxqueues); +} + + +#define EM_CAPS \ + IFCAP_TSO4 | IFCAP_TXCSUM | IFCAP_LRO | IFCAP_RXCSUM | IFCAP_VLAN_HWFILTER | IFCAP_WOL_MAGIC | \ + IFCAP_WOL_MCAST | IFCAP_WOL | IFCAP_VLAN_HWTSO | IFCAP_HWCSUM | IFCAP_VLAN_HWTAGGING | \ + IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWTSO | IFCAP_VLAN_MTU; + +#define IGB_CAPS \ + IFCAP_TSO4 | IFCAP_TXCSUM | IFCAP_LRO | IFCAP_RXCSUM | IFCAP_VLAN_HWFILTER | IFCAP_WOL_MAGIC | \ + IFCAP_WOL_MCAST | IFCAP_WOL | IFCAP_VLAN_HWTSO | IFCAP_HWCSUM | IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM | \ + IFCAP_VLAN_HWTSO | IFCAP_VLAN_MTU | IFCAP_TXCSUM_IPV6 | IFCAP_HWCSUM_IPV6 | IFCAP_JUMBO_MTU; + /********************************************************************* * Device initialization routine * @@ -509,23 +702,30 @@ em_probe(device_t dev) *********************************************************************/ static int -em_attach(device_t dev) +em_if_attach_pre(if_ctx_t ctx) { - struct adapter *adapter; - struct e1000_hw *hw; - int error = 0; + struct adapter *adapter; + if_softc_ctx_t scctx; + device_t dev; + struct e1000_hw *hw; + int error = 0; - INIT_DEBUGOUT("em_attach: begin"); + INIT_DEBUGOUT("em_if_attach_pre begin"); + dev = iflib_get_dev(ctx); + adapter = iflib_get_softc(ctx); if (resource_disabled("em", device_get_unit(dev))) { device_printf(dev, "Disabled by device hint\n"); return (ENXIO); } - adapter = device_get_softc(dev); + adapter->ctx = ctx; adapter->dev = adapter->osdep.dev = dev; + scctx = adapter->shared = iflib_get_softc_ctx(ctx); + adapter->media = iflib_get_media(ctx); hw = &adapter->hw; - EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); + + adapter->tx_process_limit = scctx->isc_ntxd[0]; /* SYSCTL stuff */ SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), @@ -543,13 +743,78 @@ em_attach(device_t dev) OID_AUTO, "fc", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, em_set_flowcntl, "I", "Flow Control"); - callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "reg_dump", CTLTYPE_STRING | CTLFLAG_RD, adapter, 0, + em_get_regs, "A", "Dump Registers"); + + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "rs_dump", CTLTYPE_INT | CTLFLAG_RW, adapter, 0, + em_get_rs, "I", "Dump RS indexes"); /* Determine hardware and mac info */ - em_identify_hardware(adapter); + em_identify_hardware(ctx); + + /* Set isc_msix_bar */ + scctx->isc_msix_bar = PCIR_BAR(EM_MSIX_BAR); + scctx->isc_tx_nsegments = EM_MAX_SCATTER; + scctx->isc_tx_tso_segments_max = scctx->isc_tx_nsegments; + scctx->isc_tx_tso_size_max = EM_TSO_SIZE; + scctx->isc_tx_tso_segsize_max = EM_TSO_SEG_SIZE; + scctx->isc_nrxqsets_max = scctx->isc_ntxqsets_max = em_set_num_queues(ctx); + device_printf(dev, "attach_pre capping queues at %d\n", scctx->isc_ntxqsets_max); + + scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO; + + + if (adapter->hw.mac.type >= igb_mac_min) { + int try_second_bar; + + scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] * sizeof(union e1000_adv_tx_desc), EM_DBA_ALIGN); + scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] * sizeof(union e1000_adv_rx_desc), EM_DBA_ALIGN); + scctx->isc_txd_size[0] = sizeof(union e1000_adv_tx_desc); + scctx->isc_rxd_size[0] = sizeof(union e1000_adv_rx_desc); + scctx->isc_txrx = &igb_txrx; + scctx->isc_capenable = IGB_CAPS; + scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_TSO | CSUM_IP6_TCP \ + | CSUM_IP6_UDP | CSUM_IP6_TCP; + if (adapter->hw.mac.type != e1000_82575) + scctx->isc_tx_csum_flags |= CSUM_SCTP | CSUM_IP6_SCTP; + + /* + ** Some new devices, as with ixgbe, now may + ** use a different BAR, so we need to keep + ** track of which is used. + */ + try_second_bar = pci_read_config(dev, scctx->isc_msix_bar, 4); + if (try_second_bar == 0) + scctx->isc_msix_bar += 4; + + } else if (adapter->hw.mac.type >= em_mac_min) { + scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0]* sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); + scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] * sizeof(union e1000_rx_desc_extended), EM_DBA_ALIGN); + scctx->isc_txd_size[0] = sizeof(struct e1000_tx_desc); + scctx->isc_rxd_size[0] = sizeof(union e1000_rx_desc_extended); + scctx->isc_txrx = &em_txrx; + scctx->isc_capenable = EM_CAPS; + scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO; + } else { + scctx->isc_txqsizes[0] = roundup2((scctx->isc_ntxd[0] + 1) * sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); + scctx->isc_rxqsizes[0] = roundup2((scctx->isc_nrxd[0] + 1) * sizeof(struct e1000_rx_desc), EM_DBA_ALIGN); + scctx->isc_txd_size[0] = sizeof(struct e1000_tx_desc); + scctx->isc_rxd_size[0] = sizeof(struct e1000_rx_desc); + scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO; + scctx->isc_txrx = &lem_txrx; + scctx->isc_capenable = EM_CAPS; + if (adapter->hw.mac.type < e1000_82543) + scctx->isc_capenable &= ~(IFCAP_HWCSUM|IFCAP_VLAN_HWCSUM); + scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO; + scctx->isc_msix_bar = 0; + } /* Setup PCI resources */ - if (em_allocate_pci_resources(adapter)) { + if (em_allocate_pci_resources(ctx)) { device_printf(dev, "Allocation of PCI resources failed\n"); error = ENXIO; goto err_pci; @@ -558,7 +823,7 @@ em_attach(device_t dev) /* ** For ICH8 and family we need to ** map the flash memory, and this - ** must happen after the MAC is + ** must happen after the MAC is ** identified */ if ((hw->mac.type == e1000_ich8lan) || @@ -605,11 +870,7 @@ em_attach(device_t dev) goto err_pci; } - /* - * Setup MSI/X or MSI if PCI Express - */ - adapter->msix = em_setup_msix(adapter); - + em_setup_msix(ctx); e1000_get_bus_info(hw); /* Set up some sysctls for the tunable interrupt delays */ @@ -635,36 +896,14 @@ em_attach(device_t dev) E1000_REGISTER(hw, E1000_ITR), DEFAULT_ITR); - /* Sysctl for limiting the amount of work done in the taskqueue */ - em_set_sysctl_value(adapter, "rx_processing_limit", - "max number of rx packets to process", &adapter->rx_process_limit, - em_rx_process_limit); - - /* - * Validate number of transmit and receive descriptors. It - * must not exceed hardware maximum, and must be multiple - * of E1000_DBA_ALIGN. - */ - if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 || - (em_txd > EM_MAX_TXD) || (em_txd < EM_MIN_TXD)) { - device_printf(dev, "Using %d TX descriptors instead of %d!\n", - EM_DEFAULT_TXD, em_txd); - adapter->num_tx_desc = EM_DEFAULT_TXD; - } else - adapter->num_tx_desc = em_txd; - - if (((em_rxd * sizeof(union e1000_rx_desc_extended)) % EM_DBA_ALIGN) != 0 || - (em_rxd > EM_MAX_RXD) || (em_rxd < EM_MIN_RXD)) { - device_printf(dev, "Using %d RX descriptors instead of %d!\n", - EM_DEFAULT_RXD, em_rxd); - adapter->num_rx_desc = EM_DEFAULT_RXD; - } else - adapter->num_rx_desc = em_rxd; - hw->mac.autoneg = DO_AUTO_NEG; hw->phy.autoneg_wait_to_complete = FALSE; hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; + if (adapter->hw.mac.type < em_mac_min) { + e1000_init_script_state_82541(&adapter->hw, TRUE); + e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE); + } /* Copper options */ if (hw->phy.media_type == e1000_media_type_copper) { hw->phy.mdix = AUTO_ALL_MODES; @@ -676,7 +915,7 @@ em_attach(device_t dev) * Set the frame limits assuming * standard ethernet sized frames. */ - adapter->hw.mac.max_frame_size = + scctx->isc_max_frame_size = adapter->hw.mac.max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; /* @@ -685,14 +924,6 @@ em_attach(device_t dev) */ hw->mac.report_tx_early = 1; - /* - ** Get queue/ring memory - */ - if (em_allocate_queues(adapter)) { - error = ENOMEM; - goto err_pci; - } - /* Allocate multicast array memory. */ adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); @@ -705,7 +936,7 @@ em_attach(device_t dev) /* Check SOL/IDER usage */ if (e1000_check_reset_block(hw)) device_printf(dev, "PHY reset is blocked" - " due to SOL/IDER session.\n"); + " due to SOL/IDER session.\n"); /* Sysctl for setting Energy Efficient Ethernet */ hw->dev_spec.ich8lan.eee_disable = eee_setting; @@ -722,7 +953,6 @@ em_attach(device_t dev) */ e1000_reset_hw(hw); - /* Make sure we have a good EEPROM before we read from it */ if (e1000_validate_nvm_checksum(hw) < 0) { /* @@ -741,7 +971,7 @@ em_attach(device_t dev) /* Copy the permanent MAC address out of the EEPROM */ if (e1000_read_mac_addr(hw) < 0) { device_printf(dev, "EEPROM read error while reading MAC" - " address\n"); + " address\n"); error = EIO; goto err_late; } @@ -756,67 +986,57 @@ em_attach(device_t dev) e1000_disable_ulp_lpt_lp(hw, TRUE); /* - ** Do interrupt configuration - */ - if (adapter->msix > 1) /* Do MSIX */ - error = em_allocate_msix(adapter); - else /* MSI or Legacy */ - error = em_allocate_legacy(adapter); - if (error) - goto err_late; - - /* * Get Wake-on-Lan and Management info for later use */ - em_get_wakeup(dev); + em_get_wakeup(ctx); + + iflib_set_mac(ctx, hw->mac.addr); + + return (0); + +err_late: + em_release_hw_control(adapter); +err_pci: + em_free_pci_resources(ctx); + free(adapter->mta, M_DEVBUF); + + return (error); +} +static int +em_if_attach_post(if_ctx_t ctx) +{ + struct adapter *adapter = iflib_get_softc(ctx); + struct e1000_hw *hw = &adapter->hw; + int error = 0; + /* Setup OS specific network interface */ - if (em_setup_interface(dev, adapter) != 0) + error = em_setup_interface(ctx); + if (error != 0) { goto err_late; + } - em_reset(adapter); + em_reset(ctx); /* Initialize statistics */ em_update_stats_counters(adapter); - hw->mac.get_link_status = 1; - em_update_link_status(adapter); - - /* Register for VLAN events */ - adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, - em_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); - adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, - em_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); - + em_if_update_admin_status(ctx); em_add_hw_stats(adapter); /* Non-AMT based hardware can now take control from firmware */ if (adapter->has_manage && !adapter->has_amt) em_get_hw_control(adapter); + + INIT_DEBUGOUT("em_if_attach_post: end"); - /* Tell the stack that the interface is not active */ - if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); - - adapter->led_dev = led_create(em_led_func, adapter, - device_get_nameunit(dev)); -#ifdef DEV_NETMAP - em_netmap_attach(adapter); -#endif /* DEV_NETMAP */ - - INIT_DEBUGOUT("em_attach: end"); - - return (0); + return (error); err_late: - em_free_transmit_structures(adapter); - em_free_receive_structures(adapter); em_release_hw_control(adapter); - if (adapter->ifp != (void *)NULL) - if_free(adapter->ifp); -err_pci: - em_free_pci_resources(adapter); + em_free_pci_resources(ctx); + em_if_queues_free(ctx); free(adapter->mta, M_DEVBUF); - EM_CORE_LOCK_DESTROY(adapter); return (error); } @@ -832,60 +1052,17 @@ err_pci: *********************************************************************/ static int -em_detach(device_t dev) +em_if_detach(if_ctx_t ctx) { - struct adapter *adapter = device_get_softc(dev); - if_t ifp = adapter->ifp; + struct adapter *adapter = iflib_get_softc(ctx); INIT_DEBUGOUT("em_detach: begin"); - /* Make sure VLANS are not using driver */ - if (if_vlantrunkinuse(ifp)) { - device_printf(dev,"Vlan in use, detach first\n"); - return (EBUSY); - } - -#ifdef DEVICE_POLLING - if (if_getcapenable(ifp) & IFCAP_POLLING) - ether_poll_deregister(ifp); -#endif - - if (adapter->led_dev != NULL) - led_destroy(adapter->led_dev); - - EM_CORE_LOCK(adapter); - adapter->in_detach = 1; - em_stop(adapter); - EM_CORE_UNLOCK(adapter); - EM_CORE_LOCK_DESTROY(adapter); - e1000_phy_hw_reset(&adapter->hw); em_release_manageability(adapter); em_release_hw_control(adapter); - - /* Unregister VLAN events */ - if (adapter->vlan_attach != NULL) - EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); - if (adapter->vlan_detach != NULL) - EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); - - ether_ifdetach(adapter->ifp); - callout_drain(&adapter->timer); - -#ifdef DEV_NETMAP - netmap_detach(ifp); -#endif /* DEV_NETMAP */ - - em_free_pci_resources(adapter); - bus_generic_detach(dev); - if_free(ifp); - - em_free_transmit_structures(adapter); - em_free_receive_structures(adapter); - - em_release_hw_control(adapter); - free(adapter->mta, M_DEVBUF); + em_free_pci_resources(ctx); return (0); } @@ -897,448 +1074,84 @@ em_detach(device_t dev) **********************************************************************/ static int -em_shutdown(device_t dev) +em_if_shutdown(if_ctx_t ctx) { - return em_suspend(dev); + return em_if_suspend(ctx); } /* * Suspend/resume device methods. */ static int -em_suspend(device_t dev) +em_if_suspend(if_ctx_t ctx) { - struct adapter *adapter = device_get_softc(dev); + struct adapter *adapter = iflib_get_softc(ctx); - EM_CORE_LOCK(adapter); - - em_release_manageability(adapter); + em_release_manageability(adapter); em_release_hw_control(adapter); - em_enable_wakeup(dev); - - EM_CORE_UNLOCK(adapter); - - return bus_generic_suspend(dev); + em_enable_wakeup(ctx); + return (0); } static int -em_resume(device_t dev) +em_if_resume(if_ctx_t ctx) { - struct adapter *adapter = device_get_softc(dev); - struct tx_ring *txr = adapter->tx_rings; - if_t ifp = adapter->ifp; + struct adapter *adapter = iflib_get_softc(ctx); - EM_CORE_LOCK(adapter); if (adapter->hw.mac.type == e1000_pch2lan) e1000_resume_workarounds_pchlan(&adapter->hw); - em_init_locked(adapter); + em_if_init(ctx); em_init_manageability(adapter); - if ((if_getflags(ifp) & IFF_UP) && - (if_getdrvflags(ifp) & IFF_DRV_RUNNING) && adapter->link_active) { - for (int i = 0; i < adapter->num_queues; i++, txr++) { - EM_TX_LOCK(txr); -#ifdef EM_MULTIQUEUE - if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr); -#else - if (!if_sendq_empty(ifp)) - em_start_locked(ifp, txr); -#endif - EM_TX_UNLOCK(txr); - } - } - EM_CORE_UNLOCK(adapter); - - return bus_generic_resume(dev); -} - - -#ifndef EM_MULTIQUEUE -static void -em_start_locked(if_t ifp, struct tx_ring *txr) -{ - struct adapter *adapter = if_getsoftc(ifp); - struct mbuf *m_head; - - EM_TX_LOCK_ASSERT(txr); - - if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != - IFF_DRV_RUNNING) - return; - - if (!adapter->link_active) - return; - - while (!if_sendq_empty(ifp)) { - /* Call cleanup if number of TX descriptors low */ - if (txr->tx_avail <= EM_TX_CLEANUP_THRESHOLD) - em_txeof(txr); - if (txr->tx_avail < EM_MAX_SCATTER) { - if_setdrvflagbits(ifp,IFF_DRV_OACTIVE, 0); - break; - } - m_head = if_dequeue(ifp); - if (m_head == NULL) - break; - /* - * Encapsulation can modify our pointer, and or make it - * NULL on failure. In that event, we can't requeue. - */ - if (em_xmit(txr, &m_head)) { - if (m_head == NULL) - break; - if_sendq_prepend(ifp, m_head); - break; - } - - /* Mark the queue as having work */ - if (txr->busy == EM_TX_IDLE) - txr->busy = EM_TX_BUSY; - - /* Send a copy of the frame to the BPF listener */ - ETHER_BPF_MTAP(ifp, m_head); - - } - - return; + return(0); } -static void -em_start(if_t ifp) -{ - struct adapter *adapter = if_getsoftc(ifp); - struct tx_ring *txr = adapter->tx_rings; - - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { - EM_TX_LOCK(txr); - em_start_locked(ifp, txr); - EM_TX_UNLOCK(txr); - } - return; -} -#else /* EM_MULTIQUEUE */ -/********************************************************************* - * Multiqueue Transmit routines - * - * em_mq_start is called by the stack to initiate a transmit. - * however, if busy the driver can queue the request rather - * than do an immediate send. It is this that is an advantage - * in this driver, rather than also having multiple tx queues. - **********************************************************************/ -/* -** Multiqueue capable stack interface -*/ static int -em_mq_start(if_t ifp, struct mbuf *m) +em_if_mtu_set(if_ctx_t ctx, uint32_t mtu) { - struct adapter *adapter = if_getsoftc(ifp); - struct tx_ring *txr = adapter->tx_rings; - unsigned int i, error; - - if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) - i = m->m_pkthdr.flowid % adapter->num_queues; - else - i = curcpu % adapter->num_queues; - - txr = &adapter->tx_rings[i]; - - error = drbr_enqueue(ifp, txr->br, m); - if (error) - return (error); + int max_frame_size; + struct adapter *adapter = iflib_get_softc(ctx); + if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx); - if (EM_TX_TRYLOCK(txr)) { - em_mq_start_locked(ifp, txr); - EM_TX_UNLOCK(txr); - } else - taskqueue_enqueue(txr->tq, &txr->tx_task); + IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); - return (0); -} - -static int -em_mq_start_locked(if_t ifp, struct tx_ring *txr) -{ - struct adapter *adapter = txr->adapter; - struct mbuf *next; - int err = 0, enq = 0; - - EM_TX_LOCK_ASSERT(txr); - - if (((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) || - adapter->link_active == 0) { - return (ENETDOWN); - } - - /* Process the queue */ - while ((next = drbr_peek(ifp, txr->br)) != NULL) { - if ((err = em_xmit(txr, &next)) != 0) { - if (next == NULL) { - /* It was freed, move forward */ - drbr_advance(ifp, txr->br); - } else { - /* - * Still have one left, it may not be - * the same since the transmit function - * may have changed it. - */ - drbr_putback(ifp, txr->br, next); - } - break; - } - drbr_advance(ifp, txr->br); - enq++; - if_inc_counter(ifp, IFCOUNTER_OBYTES, next->m_pkthdr.len); - if (next->m_flags & M_MCAST) - if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); - ETHER_BPF_MTAP(ifp, next); - if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) - break; - } - - /* Mark the queue as having work */ - if ((enq > 0) && (txr->busy == EM_TX_IDLE)) - txr->busy = EM_TX_BUSY; - - if (txr->tx_avail < EM_MAX_SCATTER) - em_txeof(txr); - if (txr->tx_avail < EM_MAX_SCATTER) { - if_setdrvflagbits(ifp, IFF_DRV_OACTIVE,0); - } - return (err); -} - -/* -** Flush all ring buffers -*/ -static void -em_qflush(if_t ifp) -{ - struct adapter *adapter = if_getsoftc(ifp); - struct tx_ring *txr = adapter->tx_rings; - struct mbuf *m; - - for (int i = 0; i < adapter->num_queues; i++, txr++) { - EM_TX_LOCK(txr); - while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) - m_freem(m); - EM_TX_UNLOCK(txr); - } - if_qflush(ifp); -} -#endif /* EM_MULTIQUEUE */ - -/********************************************************************* - * Ioctl entry point - * - * em_ioctl is called when the user wants to configure the - * interface. - * - * return 0 on success, positive on failure - **********************************************************************/ - -static int -em_ioctl(if_t ifp, u_long command, caddr_t data) -{ - struct adapter *adapter = if_getsoftc(ifp); - struct ifreq *ifr = (struct ifreq *)data; -#if defined(INET) || defined(INET6) - struct ifaddr *ifa = (struct ifaddr *)data; -#endif - bool avoid_reset = FALSE; - int error = 0; - - if (adapter->in_detach) - return (error); - - switch (command) { - case SIOCSIFADDR: -#ifdef INET - if (ifa->ifa_addr->sa_family == AF_INET) - avoid_reset = TRUE; -#endif -#ifdef INET6 - if (ifa->ifa_addr->sa_family == AF_INET6) - avoid_reset = TRUE; -#endif - /* - ** Calling init results in link renegotiation, - ** so we avoid doing it when possible. - */ - if (avoid_reset) { - if_setflagbits(ifp,IFF_UP,0); - if (!(if_getdrvflags(ifp)& IFF_DRV_RUNNING)) - em_init(adapter); -#ifdef INET - if (!(if_getflags(ifp) & IFF_NOARP)) - arp_ifinit(ifp, ifa); -#endif - } else - error = ether_ioctl(ifp, command, data); - break; - case SIOCSIFMTU: - { - int max_frame_size; - - IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); - - EM_CORE_LOCK(adapter); - switch (adapter->hw.mac.type) { - case e1000_82571: - case e1000_82572: - case e1000_ich9lan: - case e1000_ich10lan: - case e1000_pch2lan: - case e1000_pch_lpt: - case e1000_pch_spt: - case e1000_82574: - case e1000_82583: - case e1000_80003es2lan: /* 9K Jumbo Frame size */ - max_frame_size = 9234; - break; - case e1000_pchlan: - max_frame_size = 4096; - break; - /* Adapters that do not support jumbo frames */ - case e1000_ich8lan: - max_frame_size = ETHER_MAX_LEN; - break; - default: - max_frame_size = MAX_JUMBO_FRAME_SIZE; - } - if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - - ETHER_CRC_LEN) { - EM_CORE_UNLOCK(adapter); - error = EINVAL; - break; - } - - if_setmtu(ifp, ifr->ifr_mtu); - adapter->hw.mac.max_frame_size = - if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN; - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) - em_init_locked(adapter); - EM_CORE_UNLOCK(adapter); - break; - } - case SIOCSIFFLAGS: - IOCTL_DEBUGOUT("ioctl rcv'd:\ - SIOCSIFFLAGS (Set Interface Flags)"); - EM_CORE_LOCK(adapter); - if (if_getflags(ifp) & IFF_UP) { - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { - if ((if_getflags(ifp) ^ adapter->if_flags) & - (IFF_PROMISC | IFF_ALLMULTI)) { - em_disable_promisc(adapter); - em_set_promisc(adapter); - } - } else - em_init_locked(adapter); - } else - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) - em_stop(adapter); - adapter->if_flags = if_getflags(ifp); - EM_CORE_UNLOCK(adapter); - break; - case SIOCADDMULTI: - case SIOCDELMULTI: - IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { - EM_CORE_LOCK(adapter); - em_disable_intr(adapter); - em_set_multi(adapter); -#ifdef DEVICE_POLLING - if (!(if_getcapenable(ifp) & IFCAP_POLLING)) -#endif - em_enable_intr(adapter); - EM_CORE_UNLOCK(adapter); - } + switch (adapter->hw.mac.type) { + case e1000_82571: + case e1000_82572: + case e1000_ich9lan: + case e1000_ich10lan: + case e1000_pch2lan: + case e1000_pch_lpt: + case e1000_pch_spt: + case e1000_82574: + case e1000_82583: + case e1000_80003es2lan: + /* 9K Jumbo Frame size */ + max_frame_size = 9234; break; - case SIOCSIFMEDIA: - /* Check SOL/IDER usage */ - EM_CORE_LOCK(adapter); - if (e1000_check_reset_block(&adapter->hw)) { - EM_CORE_UNLOCK(adapter); - device_printf(adapter->dev, "Media change is" - " blocked due to SOL/IDER session.\n"); - break; - } - EM_CORE_UNLOCK(adapter); - /* falls thru */ - case SIOCGIFMEDIA: - IOCTL_DEBUGOUT("ioctl rcv'd: \ - SIOCxIFMEDIA (Get/Set Interface Media)"); - error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); + case e1000_pchlan: + max_frame_size = 4096; break; - case SIOCSIFCAP: - { - int mask, reinit; - - IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); - reinit = 0; - mask = ifr->ifr_reqcap ^ if_getcapenable(ifp); -#ifdef DEVICE_POLLING - if (mask & IFCAP_POLLING) { - if (ifr->ifr_reqcap & IFCAP_POLLING) { - error = ether_poll_register(em_poll, ifp); - if (error) - return (error); - EM_CORE_LOCK(adapter); - em_disable_intr(adapter); - if_setcapenablebit(ifp, IFCAP_POLLING, 0); - EM_CORE_UNLOCK(adapter); - } else { - error = ether_poll_deregister(ifp); - /* Enable interrupt even in error case */ - EM_CORE_LOCK(adapter); - em_enable_intr(adapter); - if_setcapenablebit(ifp, 0, IFCAP_POLLING); - EM_CORE_UNLOCK(adapter); - } - } -#endif - if (mask & IFCAP_HWCSUM) { - if_togglecapenable(ifp,IFCAP_HWCSUM); - reinit = 1; - } - if (mask & IFCAP_TSO4) { - if_togglecapenable(ifp,IFCAP_TSO4); - reinit = 1; - } - if (mask & IFCAP_VLAN_HWTAGGING) { - if_togglecapenable(ifp,IFCAP_VLAN_HWTAGGING); - reinit = 1; - } - if (mask & IFCAP_VLAN_HWFILTER) { - if_togglecapenable(ifp, IFCAP_VLAN_HWFILTER); - reinit = 1; - } - if (mask & IFCAP_VLAN_HWTSO) { - if_togglecapenable(ifp, IFCAP_VLAN_HWTSO); - reinit = 1; - } - if ((mask & IFCAP_WOL) && - (if_getcapabilities(ifp) & IFCAP_WOL) != 0) { - if (mask & IFCAP_WOL_MCAST) - if_togglecapenable(ifp, IFCAP_WOL_MCAST); - if (mask & IFCAP_WOL_MAGIC) - if_togglecapenable(ifp, IFCAP_WOL_MAGIC); - } - if (reinit && (if_getdrvflags(ifp) & IFF_DRV_RUNNING)) - em_init(adapter); - if_vlancap(ifp); + case e1000_82542: + case e1000_ich8lan: + /* Adapters that do not support jumbo frames */ + max_frame_size = ETHER_MAX_LEN; break; - } - default: - error = ether_ioctl(ifp, command, data); - break; + if (adapter->hw.mac.type >= igb_mac_min) + max_frame_size = 9234; + else /* lem */ + max_frame_size = MAX_JUMBO_FRAME_SIZE; + } + if (mtu > max_frame_size - ETHER_HDR_LEN - ETHER_CRC_LEN) { + return (EINVAL); } - return (error); + scctx->isc_max_frame_size = adapter->hw.mac.max_frame_size = + mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; + return (0); } - /********************************************************************* * Init entry point * @@ -1351,21 +1164,17 @@ em_ioctl(if_t ifp, u_long command, caddr_t data) **********************************************************************/ static void -em_init_locked(struct adapter *adapter) +em_if_init(if_ctx_t ctx) { - if_t ifp = adapter->ifp; - device_t dev = adapter->dev; - - INIT_DEBUGOUT("em_init: begin"); - - EM_CORE_LOCK_ASSERT(adapter); - - em_disable_intr(adapter); - callout_stop(&adapter->timer); + struct adapter *adapter = iflib_get_softc(ctx); + struct ifnet *ifp = iflib_get_ifp(ctx); + struct em_tx_queue *tx_que; + int i; + INIT_DEBUGOUT("em_if_init: begin"); /* Get the latest mac address, User can use a LAA */ - bcopy(if_getlladdr(adapter->ifp), adapter->hw.mac.addr, - ETHER_ADDR_LEN); + bcopy(if_getlladdr(ifp), adapter->hw.mac.addr, + ETHER_ADDR_LEN); /* Put the address into the Receive Address Array */ e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); @@ -1382,49 +1191,49 @@ em_init_locked(struct adapter *adapter) E1000_RAR_ENTRIES - 1); } + /* Initialize the hardware */ - em_reset(adapter); - em_update_link_status(adapter); + em_reset(ctx); + em_if_update_admin_status(ctx); + + for (i = 0, tx_que = adapter->tx_queues; i < adapter->tx_num_queues; i++, tx_que++) { + struct tx_ring *txr = &tx_que->txr; + + txr->tx_rs_cidx = txr->tx_rs_pidx = txr->tx_cidx_processed = 0; + } /* Setup VLAN support, basic and offload if available */ E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); - /* Set hardware offload abilities */ - if_clearhwassist(ifp); - if (if_getcapenable(ifp) & IFCAP_TXCSUM) - if_sethwassistbits(ifp, CSUM_TCP | CSUM_UDP, 0); - - if (if_getcapenable(ifp) & IFCAP_TSO4) - if_sethwassistbits(ifp, CSUM_TSO, 0); + /* Clear bad data from Rx FIFOs */ + if (adapter->hw.mac.type >= igb_mac_min) + e1000_rx_fifo_flush_82575(&adapter->hw); /* Configure for OS presence */ em_init_manageability(adapter); /* Prepare transmit descriptors and buffers */ - em_setup_transmit_structures(adapter); - em_initialize_transmit_unit(adapter); + em_initialize_transmit_unit(ctx); /* Setup Multicast table */ - em_set_multi(adapter); + em_if_multi_set(ctx); /* - ** Figure out the desired mbuf - ** pool for doing jumbos - */ + * Figure out the desired mbuf + * pool for doing jumbos + */ if (adapter->hw.mac.max_frame_size <= 2048) adapter->rx_mbuf_sz = MCLBYTES; +#ifndef CONTIGMALLOC_WORKS + else + adapter->rx_mbuf_sz = MJUMPAGESIZE; +#else else if (adapter->hw.mac.max_frame_size <= 4096) adapter->rx_mbuf_sz = MJUMPAGESIZE; else adapter->rx_mbuf_sz = MJUM9BYTES; - - /* Prepare receive descriptors and buffers */ - if (em_setup_receive_structures(adapter)) { - device_printf(dev, "Could not setup receive structures\n"); - em_stop(adapter); - return; - } - em_initialize_receive_unit(adapter); +#endif + em_initialize_receive_unit(ctx); /* Use real VLAN Filter support? */ if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) { @@ -1440,123 +1249,59 @@ em_init_locked(struct adapter *adapter) } /* Don't lose promiscuous settings */ - em_set_promisc(adapter); - - /* Set the interface as ACTIVE */ - if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); - - callout_reset(&adapter->timer, hz, em_local_timer, adapter); + em_if_set_promisc(ctx, IFF_PROMISC); e1000_clear_hw_cntrs_base_generic(&adapter->hw); /* MSI/X configuration for 82574 */ if (adapter->hw.mac.type == e1000_82574) { - int tmp; - tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); + int tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); + tmp |= E1000_CTRL_EXT_PBA_CLR; E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp); /* Set the IVAR - interrupt vector routing. */ E1000_WRITE_REG(&adapter->hw, E1000_IVAR, adapter->ivars); - } + } else if (adapter->intr_type == IFLIB_INTR_MSIX) /* Set up queue routing */ + igb_configure_queues(adapter); -#ifdef DEVICE_POLLING - /* - * Only enable interrupts if we are not polling, make sure - * they are off otherwise. - */ - if (if_getcapenable(ifp) & IFCAP_POLLING) - em_disable_intr(adapter); - else -#endif /* DEVICE_POLLING */ - em_enable_intr(adapter); + /* this clears any pending interrupts */ + E1000_READ_REG(&adapter->hw, E1000_ICR); + E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC); /* AMT based hardware can now take control from firmware */ if (adapter->has_manage && adapter->has_amt) em_get_hw_control(adapter); -} - -static void -em_init(void *arg) -{ - struct adapter *adapter = arg; - - EM_CORE_LOCK(adapter); - em_init_locked(adapter); - EM_CORE_UNLOCK(adapter); -} - - -#ifdef DEVICE_POLLING -/********************************************************************* - * - * Legacy polling routine: note this only works with single queue - * - *********************************************************************/ -static int -em_poll(if_t ifp, enum poll_cmd cmd, int count) -{ - struct adapter *adapter = if_getsoftc(ifp); - struct tx_ring *txr = adapter->tx_rings; - struct rx_ring *rxr = adapter->rx_rings; - u32 reg_icr; - int rx_done; - - EM_CORE_LOCK(adapter); - if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) { - EM_CORE_UNLOCK(adapter); - return (0); - } - if (cmd == POLL_AND_CHECK_STATUS) { - reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); - if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { - callout_stop(&adapter->timer); - adapter->hw.mac.get_link_status = 1; - em_update_link_status(adapter); - callout_reset(&adapter->timer, hz, - em_local_timer, adapter); - } + /* Set Energy Efficient Ethernet */ + if (adapter->hw.mac.type >= igb_mac_min && + adapter->hw.phy.media_type == e1000_media_type_copper) { + if (adapter->hw.mac.type == e1000_i354) + e1000_set_eee_i354(&adapter->hw, TRUE, TRUE); + else + e1000_set_eee_i350(&adapter->hw, TRUE, TRUE); } - EM_CORE_UNLOCK(adapter); - - em_rxeof(rxr, count, &rx_done); - - EM_TX_LOCK(txr); - em_txeof(txr); -#ifdef EM_MULTIQUEUE - if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr); -#else - if (!if_sendq_empty(ifp)) - em_start_locked(ifp, txr); -#endif - EM_TX_UNLOCK(txr); - - return (rx_done); } -#endif /* DEVICE_POLLING */ - /********************************************************************* * - * Fast Legacy/MSI Combined Interrupt Service routine + * Fast Legacy/MSI Combined Interrupt Service routine * *********************************************************************/ -static int -em_irq_fast(void *arg) +int +em_intr(void *arg) { - struct adapter *adapter = arg; - if_t ifp; - u32 reg_icr; - - ifp = adapter->ifp; + struct adapter *adapter = arg; + if_ctx_t ctx = adapter->ctx; + u32 reg_icr; reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); - /* Hot eject? */ + if (adapter->intr_type != IFLIB_INTR_LEGACY) + goto skip_stray; + /* Hot eject? */ if (reg_icr == 0xffffffff) return FILTER_STRAY; - /* Definitely not our interrupt. */ + /* Definitely not our interrupt. */ if (reg_icr == 0x0) return FILTER_STRAY; @@ -1568,80 +1313,67 @@ em_irq_fast(void *arg) (reg_icr & E1000_ICR_INT_ASSERTED) == 0) return FILTER_STRAY; - em_disable_intr(adapter); - taskqueue_enqueue(adapter->tq, &adapter->que_task); - +skip_stray: /* Link status change */ if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { adapter->hw.mac.get_link_status = 1; - taskqueue_enqueue(taskqueue_fast, &adapter->link_task); + iflib_admin_intr_deferred(ctx); } if (reg_icr & E1000_ICR_RXO) adapter->rx_overruns++; - return FILTER_HANDLED; + + return (FILTER_SCHEDULE_THREAD); } -/* Combined RX/TX handler, used by Legacy and MSI */ static void -em_handle_que(void *context, int pending) +igb_rx_enable_queue(struct adapter *adapter, struct em_rx_queue *rxq) { - struct adapter *adapter = context; - if_t ifp = adapter->ifp; - struct tx_ring *txr = adapter->tx_rings; - struct rx_ring *rxr = adapter->rx_rings; - - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { - bool more = em_rxeof(rxr, adapter->rx_process_limit, NULL); - - EM_TX_LOCK(txr); - em_txeof(txr); -#ifdef EM_MULTIQUEUE - if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr); -#else - if (!if_sendq_empty(ifp)) - em_start_locked(ifp, txr); -#endif - EM_TX_UNLOCK(txr); - if (more) { - taskqueue_enqueue(adapter->tq, &adapter->que_task); - return; - } - } + E1000_WRITE_REG(&adapter->hw, E1000_EIMS, rxq->eims); +} - em_enable_intr(adapter); - return; +static void +em_rx_enable_queue(struct adapter *adapter, struct em_rx_queue *rxq) +{ + E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxq->eims); } +static void +igb_tx_enable_queue(struct adapter *adapter, struct em_tx_queue *txq) +{ + E1000_WRITE_REG(&adapter->hw, E1000_EIMS, txq->eims); +} -/********************************************************************* - * - * MSIX Interrupt Service Routines - * - **********************************************************************/ static void -em_msix_tx(void *arg) +em_tx_enable_queue(struct adapter *adapter, struct em_tx_queue *txq) { - struct tx_ring *txr = arg; - struct adapter *adapter = txr->adapter; - if_t ifp = adapter->ifp; - - ++txr->tx_irq; - EM_TX_LOCK(txr); - em_txeof(txr); -#ifdef EM_MULTIQUEUE - if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr); -#else - if (!if_sendq_empty(ifp)) - em_start_locked(ifp, txr); -#endif + E1000_WRITE_REG(&adapter->hw, E1000_IMS, txq->eims); +} - /* Reenable this interrupt */ - E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims); - EM_TX_UNLOCK(txr); - return; +static int +em_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) +{ + struct adapter *adapter = iflib_get_softc(ctx); + struct em_rx_queue *rxq = &adapter->rx_queues[rxqid]; + + if (adapter->hw.mac.type >= igb_mac_min) + igb_rx_enable_queue(adapter, rxq); + else + em_rx_enable_queue(adapter, rxq); + return (0); +} + +static int +em_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid) +{ + struct adapter *adapter = iflib_get_softc(ctx); + struct em_tx_queue *txq = &adapter->tx_queues[txqid]; + + if (adapter->hw.mac.type >= igb_mac_min) + igb_tx_enable_queue(adapter, txq); + else + em_tx_enable_queue(adapter, txq); + return (0); } /********************************************************************* @@ -1649,25 +1381,14 @@ em_msix_tx(void *arg) * MSIX RX Interrupt Service routine * **********************************************************************/ - -static void -em_msix_rx(void *arg) +static int +em_msix_que(void *arg) { - struct rx_ring *rxr = arg; - struct adapter *adapter = rxr->adapter; - bool more; + struct em_rx_queue *que = arg; - ++rxr->rx_irq; - if (!(if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING)) - return; - more = em_rxeof(rxr, adapter->rx_process_limit, NULL); - if (more) - taskqueue_enqueue(rxr->tq, &rxr->rx_task); - else { - /* Reenable this interrupt */ - E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims); - } - return; + ++que->irqs; + + return (FILTER_SCHEDULE_THREAD); } /********************************************************************* @@ -1675,103 +1396,50 @@ em_msix_rx(void *arg) * MSIX Link Fast Interrupt Service routine * **********************************************************************/ -static void +static int em_msix_link(void *arg) { - struct adapter *adapter = arg; - u32 reg_icr; + struct adapter *adapter = arg; + u32 reg_icr; ++adapter->link_irq; + MPASS(adapter->hw.back != NULL); reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); if (reg_icr & E1000_ICR_RXO) adapter->rx_overruns++; if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { - adapter->hw.mac.get_link_status = 1; - em_handle_link(adapter, 0); - } else + em_handle_link(adapter->ctx); + } else { E1000_WRITE_REG(&adapter->hw, E1000_IMS, - EM_MSIX_LINK | E1000_IMS_LSC); + EM_MSIX_LINK | E1000_IMS_LSC); + if (adapter->hw.mac.type >= igb_mac_min) + E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask); + } + /* - ** Because we must read the ICR for this interrupt - ** it may clear other causes using autoclear, for - ** this reason we simply create a soft interrupt - ** for all these vectors. - */ - if (reg_icr) { + * Because we must read the ICR for this interrupt + * it may clear other causes using autoclear, for + * this reason we simply create a soft interrupt + * for all these vectors. + */ + if (reg_icr && adapter->hw.mac.type < igb_mac_min) { E1000_WRITE_REG(&adapter->hw, E1000_ICS, adapter->ims); } - return; -} - -static void -em_handle_rx(void *context, int pending) -{ - struct rx_ring *rxr = context; - struct adapter *adapter = rxr->adapter; - bool more; - more = em_rxeof(rxr, adapter->rx_process_limit, NULL); - if (more) - taskqueue_enqueue(rxr->tq, &rxr->rx_task); - else { - /* Reenable this interrupt */ - E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims); - } + return (FILTER_HANDLED); } static void -em_handle_tx(void *context, int pending) +em_handle_link(void *context) { - struct tx_ring *txr = context; - struct adapter *adapter = txr->adapter; - if_t ifp = adapter->ifp; - - EM_TX_LOCK(txr); - em_txeof(txr); -#ifdef EM_MULTIQUEUE - if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr); -#else - if (!if_sendq_empty(ifp)) - em_start_locked(ifp, txr); -#endif - E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims); - EM_TX_UNLOCK(txr); -} + if_ctx_t ctx = context; + struct adapter *adapter = iflib_get_softc(ctx); -static void -em_handle_link(void *context, int pending) -{ - struct adapter *adapter = context; - struct tx_ring *txr = adapter->tx_rings; - if_t ifp = adapter->ifp; - - if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) - return; - - EM_CORE_LOCK(adapter); - callout_stop(&adapter->timer); - em_update_link_status(adapter); - callout_reset(&adapter->timer, hz, em_local_timer, adapter); - E1000_WRITE_REG(&adapter->hw, E1000_IMS, - EM_MSIX_LINK | E1000_IMS_LSC); - if (adapter->link_active) { - for (int i = 0; i < adapter->num_queues; i++, txr++) { - EM_TX_LOCK(txr); -#ifdef EM_MULTIQUEUE - if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr); -#else - if (if_sendq_empty(ifp)) - em_start_locked(ifp, txr); -#endif - EM_TX_UNLOCK(txr); - } - } - EM_CORE_UNLOCK(adapter); + adapter->hw.mac.get_link_status = 1; + iflib_admin_intr_deferred(ctx); } @@ -1784,21 +1452,19 @@ em_handle_link(void *context, int pending) * **********************************************************************/ static void -em_media_status(if_t ifp, struct ifmediareq *ifmr) +em_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) { - struct adapter *adapter = if_getsoftc(ifp); + struct adapter *adapter = iflib_get_softc(ctx); u_char fiber_type = IFM_1000_SX; - INIT_DEBUGOUT("em_media_status: begin"); + INIT_DEBUGOUT("em_if_media_status: begin"); - EM_CORE_LOCK(adapter); - em_update_link_status(adapter); + iflib_admin_intr_deferred(ctx); ifmr->ifm_status = IFM_AVALID; ifmr->ifm_active = IFM_ETHER; if (!adapter->link_active) { - EM_CORE_UNLOCK(adapter); return; } @@ -1806,6 +1472,8 @@ em_media_status(if_t ifp, struct ifmediareq *ifmr) if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { + if (adapter->hw.mac.type == e1000_82545) + fiber_type = IFM_1000_LX; ifmr->ifm_active |= fiber_type | IFM_FDX; } else { switch (adapter->link_speed) { @@ -1824,7 +1492,6 @@ em_media_status(if_t ifp, struct ifmediareq *ifmr) else ifmr->ifm_active |= IFM_HDX; } - EM_CORE_UNLOCK(adapter); } /********************************************************************* @@ -1836,17 +1503,16 @@ em_media_status(if_t ifp, struct ifmediareq *ifmr) * **********************************************************************/ static int -em_media_change(if_t ifp) +em_if_media_change(if_ctx_t ctx) { - struct adapter *adapter = if_getsoftc(ifp); - struct ifmedia *ifm = &adapter->media; + struct adapter *adapter = iflib_get_softc(ctx); + struct ifmedia *ifm = iflib_get_media(ctx); - INIT_DEBUGOUT("em_media_change: begin"); + INIT_DEBUGOUT("em_if_media_change: begin"); if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) return (EINVAL); - EM_CORE_LOCK(adapter); switch (IFM_SUBTYPE(ifm->ifm_media)) { case IFM_AUTO: adapter->hw.mac.autoneg = DO_AUTO_NEG; @@ -1878,361 +1544,45 @@ em_media_change(if_t ifp) device_printf(adapter->dev, "Unsupported media type\n"); } - em_init_locked(adapter); - EM_CORE_UNLOCK(adapter); + em_if_init(ctx); return (0); } -/********************************************************************* - * - * This routine maps the mbufs to tx descriptors. - * - * return 0 on success, positive on failure - **********************************************************************/ - static int -em_xmit(struct tx_ring *txr, struct mbuf **m_headp) +em_if_set_promisc(if_ctx_t ctx, int flags) { - struct adapter *adapter = txr->adapter; - bus_dma_segment_t segs[EM_MAX_SCATTER]; - bus_dmamap_t map; - struct em_txbuffer *tx_buffer, *tx_buffer_mapped; - struct e1000_tx_desc *ctxd = NULL; - struct mbuf *m_head; - struct ether_header *eh; - struct ip *ip = NULL; - struct tcphdr *tp = NULL; - u32 txd_upper = 0, txd_lower = 0; - int ip_off, poff; - int nsegs, i, j, first, last = 0; - int error; - bool do_tso, tso_desc, remap = TRUE; - - m_head = *m_headp; - do_tso = (m_head->m_pkthdr.csum_flags & CSUM_TSO); - tso_desc = FALSE; - ip_off = poff = 0; - - /* - * Intel recommends entire IP/TCP header length reside in a single - * buffer. If multiple descriptors are used to describe the IP and - * TCP header, each descriptor should describe one or more - * complete headers; descriptors referencing only parts of headers - * are not supported. If all layer headers are not coalesced into - * a single buffer, each buffer should not cross a 4KB boundary, - * or be larger than the maximum read request size. - * Controller also requires modifing IP/TCP header to make TSO work - * so we firstly get a writable mbuf chain then coalesce ethernet/ - * IP/TCP header into a single buffer to meet the requirement of - * controller. This also simplifies IP/TCP/UDP checksum offloading - * which also has similar restrictions. - */ - if (do_tso || m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) { - if (do_tso || (m_head->m_next != NULL && - m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)) { - if (M_WRITABLE(*m_headp) == 0) { - m_head = m_dup(*m_headp, M_NOWAIT); - m_freem(*m_headp); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); - } - *m_headp = m_head; - } - } - /* - * XXX - * Assume IPv4, we don't have TSO/checksum offload support - * for IPv6 yet. - */ - ip_off = sizeof(struct ether_header); - if (m_head->m_len < ip_off) { - m_head = m_pullup(m_head, ip_off); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); - } - } - eh = mtod(m_head, struct ether_header *); - if (eh->ether_type == htons(ETHERTYPE_VLAN)) { - ip_off = sizeof(struct ether_vlan_header); - if (m_head->m_len < ip_off) { - m_head = m_pullup(m_head, ip_off); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); - } - } - } - if (m_head->m_len < ip_off + sizeof(struct ip)) { - m_head = m_pullup(m_head, ip_off + sizeof(struct ip)); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); - } - } - ip = (struct ip *)(mtod(m_head, char *) + ip_off); - poff = ip_off + (ip->ip_hl << 2); - - if (do_tso || (m_head->m_pkthdr.csum_flags & CSUM_TCP)) { - if (m_head->m_len < poff + sizeof(struct tcphdr)) { - m_head = m_pullup(m_head, poff + - sizeof(struct tcphdr)); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); - } - } - tp = (struct tcphdr *)(mtod(m_head, char *) + poff); - /* - * TSO workaround: - * pull 4 more bytes of data into it. - */ - if (m_head->m_len < poff + (tp->th_off << 2)) { - m_head = m_pullup(m_head, poff + - (tp->th_off << 2) + - TSO_WORKAROUND); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); - } - } - ip = (struct ip *)(mtod(m_head, char *) + ip_off); - tp = (struct tcphdr *)(mtod(m_head, char *) + poff); - if (do_tso) { - ip->ip_len = htons(m_head->m_pkthdr.tso_segsz + - (ip->ip_hl << 2) + - (tp->th_off << 2)); - ip->ip_sum = 0; - /* - * The pseudo TCP checksum does not include TCP - * payload length so driver should recompute - * the checksum here what hardware expect to - * see. This is adherence of Microsoft's Large - * Send specification. - */ - tp->th_sum = in_pseudo(ip->ip_src.s_addr, - ip->ip_dst.s_addr, htons(IPPROTO_TCP)); - } - } else if (m_head->m_pkthdr.csum_flags & CSUM_UDP) { - if (m_head->m_len < poff + sizeof(struct udphdr)) { - m_head = m_pullup(m_head, poff + - sizeof(struct udphdr)); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); - } - } - ip = (struct ip *)(mtod(m_head, char *) + ip_off); - } - *m_headp = m_head; - } - - /* - * Map the packet for DMA - * - * Capture the first descriptor index, - * this descriptor will have the index - * of the EOP which is the only one that - * now gets a DONE bit writeback. - */ - first = txr->next_avail_desc; - tx_buffer = &txr->tx_buffers[first]; - tx_buffer_mapped = tx_buffer; - map = tx_buffer->map; - -retry: - error = bus_dmamap_load_mbuf_sg(txr->txtag, map, - *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); - - /* - * There are two types of errors we can (try) to handle: - * - EFBIG means the mbuf chain was too long and bus_dma ran - * out of segments. Defragment the mbuf chain and try again. - * - ENOMEM means bus_dma could not obtain enough bounce buffers - * at this point in time. Defer sending and try again later. - * All other errors, in particular EINVAL, are fatal and prevent the - * mbuf chain from ever going through. Drop it and report error. - */ - if (error == EFBIG && remap) { - struct mbuf *m; - - m = m_collapse(*m_headp, M_NOWAIT, EM_MAX_SCATTER); - if (m == NULL) { - adapter->mbuf_defrag_failed++; - m_freem(*m_headp); - *m_headp = NULL; - return (ENOBUFS); - } - *m_headp = m; - - /* Try it again, but only once */ - remap = FALSE; - goto retry; - } else if (error != 0) { - adapter->no_tx_dma_setup++; - m_freem(*m_headp); - *m_headp = NULL; - return (error); - } - - /* - * TSO Hardware workaround, if this packet is not - * TSO, and is only a single descriptor long, and - * it follows a TSO burst, then we need to add a - * sentinel descriptor to prevent premature writeback. - */ - if ((!do_tso) && (txr->tx_tso == TRUE)) { - if (nsegs == 1) - tso_desc = TRUE; - txr->tx_tso = FALSE; - } - - if (txr->tx_avail < (nsegs + EM_MAX_SCATTER)) { - txr->no_desc_avail++; - bus_dmamap_unload(txr->txtag, map); - return (ENOBUFS); - } - m_head = *m_headp; - - /* Do hardware assists */ - if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { - em_tso_setup(txr, m_head, ip_off, ip, tp, - &txd_upper, &txd_lower); - /* we need to make a final sentinel transmit desc */ - tso_desc = TRUE; - } else if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) - em_transmit_checksum_setup(txr, m_head, - ip_off, ip, &txd_upper, &txd_lower); - - if (m_head->m_flags & M_VLANTAG) { - /* Set the vlan id. */ - txd_upper |= htole16(if_getvtag(m_head)) << 16; - /* Tell hardware to add tag */ - txd_lower |= htole32(E1000_TXD_CMD_VLE); - } - - i = txr->next_avail_desc; - - /* Set up our transmit descriptors */ - for (j = 0; j < nsegs; j++) { - bus_size_t seg_len; - bus_addr_t seg_addr; - - tx_buffer = &txr->tx_buffers[i]; - ctxd = &txr->tx_base[i]; - seg_addr = segs[j].ds_addr; - seg_len = segs[j].ds_len; - /* - ** TSO Workaround: - ** If this is the last descriptor, we want to - ** split it so we have a small final sentinel - */ - if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) { - seg_len -= TSO_WORKAROUND; - ctxd->buffer_addr = htole64(seg_addr); - ctxd->lower.data = htole32( - adapter->txd_cmd | txd_lower | seg_len); - ctxd->upper.data = htole32(txd_upper); - if (++i == adapter->num_tx_desc) - i = 0; - - /* Now make the sentinel */ - txr->tx_avail--; - ctxd = &txr->tx_base[i]; - tx_buffer = &txr->tx_buffers[i]; - ctxd->buffer_addr = - htole64(seg_addr + seg_len); - ctxd->lower.data = htole32( - adapter->txd_cmd | txd_lower | TSO_WORKAROUND); - ctxd->upper.data = - htole32(txd_upper); - last = i; - if (++i == adapter->num_tx_desc) - i = 0; - } else { - ctxd->buffer_addr = htole64(seg_addr); - ctxd->lower.data = htole32( - adapter->txd_cmd | txd_lower | seg_len); - ctxd->upper.data = htole32(txd_upper); - last = i; - if (++i == adapter->num_tx_desc) - i = 0; - } - tx_buffer->m_head = NULL; - tx_buffer->next_eop = -1; - } - - txr->next_avail_desc = i; - txr->tx_avail -= nsegs; + struct adapter *adapter = iflib_get_softc(ctx); + u32 reg_rctl; - tx_buffer->m_head = m_head; - /* - ** Here we swap the map so the last descriptor, - ** which gets the completion interrupt has the - ** real map, and the first descriptor gets the - ** unused map from this descriptor. - */ - tx_buffer_mapped->map = tx_buffer->map; - tx_buffer->map = map; - bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); - - /* - * Last Descriptor of Packet - * needs End Of Packet (EOP) - * and Report Status (RS) - */ - ctxd->lower.data |= - htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS); - /* - * Keep track in the first buffer which - * descriptor will be written back - */ - tx_buffer = &txr->tx_buffers[first]; - tx_buffer->next_eop = last; - - /* - * Advance the Transmit Descriptor Tail (TDT), this tells the E1000 - * that this frame is available to transmit. - */ - bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i); - - return (0); -} - -static void -em_set_promisc(struct adapter *adapter) -{ - if_t ifp = adapter->ifp; - u32 reg_rctl; + em_disable_promisc(ctx); reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); - if (if_getflags(ifp) & IFF_PROMISC) { + if (flags & IFF_PROMISC) { reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); /* Turn this on if you want to see bad packets */ if (em_debug_sbp) reg_rctl |= E1000_RCTL_SBP; E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); - } else if (if_getflags(ifp) & IFF_ALLMULTI) { + } else if (flags & IFF_ALLMULTI) { reg_rctl |= E1000_RCTL_MPE; reg_rctl &= ~E1000_RCTL_UPE; E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); } + return (0); } static void -em_disable_promisc(struct adapter *adapter) +em_disable_promisc(if_ctx_t ctx) { - if_t ifp = adapter->ifp; - u32 reg_rctl; - int mcnt = 0; + struct adapter *adapter = iflib_get_softc(ctx); + struct ifnet *ifp = iflib_get_ifp(ctx); + u32 reg_rctl; + int mcnt = 0; reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); - reg_rctl &= (~E1000_RCTL_UPE); + reg_rctl &= (~E1000_RCTL_UPE); if (if_getflags(ifp) & IFF_ALLMULTI) mcnt = MAX_NUM_MULTICAST_ADDRESSES; else @@ -2253,9 +1603,10 @@ em_disable_promisc(struct adapter *adapter) **********************************************************************/ static void -em_set_multi(struct adapter *adapter) +em_if_multi_set(if_ctx_t ctx) { - if_t ifp = adapter->ifp; + struct adapter *adapter = iflib_get_softc(ctx); + struct ifnet *ifp = iflib_get_ifp(ctx); u32 reg_rctl = 0; u8 *mta; /* Multicast array memory */ int mcnt = 0; @@ -2265,7 +1616,7 @@ em_set_multi(struct adapter *adapter) mta = adapter->mta; bzero(mta, sizeof(u8) * ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES); - if (adapter->hw.mac.type == e1000_82542 && + if (adapter->hw.mac.type == e1000_82542 && adapter->hw.revision_id == E1000_REVISION_2) { reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) @@ -2284,7 +1635,7 @@ em_set_multi(struct adapter *adapter) } else e1000_update_mc_addr_list(&adapter->hw, mta, mcnt); - if (adapter->hw.mac.type == e1000_82542 && + if (adapter->hw.mac.type == e1000_82542 && adapter->hw.revision_id == E1000_REVISION_2) { reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); reg_rctl &= ~E1000_RCTL_RST; @@ -2304,17 +1655,17 @@ em_set_multi(struct adapter *adapter) **********************************************************************/ static void -em_local_timer(void *arg) +em_if_timer(if_ctx_t ctx, uint16_t qid) { - struct adapter *adapter = arg; - if_t ifp = adapter->ifp; - struct tx_ring *txr = adapter->tx_rings; - struct rx_ring *rxr = adapter->rx_rings; - u32 trigger = 0; + struct adapter *adapter = iflib_get_softc(ctx); + struct em_rx_queue *que; + int i; + int trigger = 0; - EM_CORE_LOCK_ASSERT(adapter); + if (qid != 0) + return; - em_update_link_status(adapter); + em_if_update_admin_status(ctx); em_update_stats_counters(adapter); /* Reset LAA into RAR[0] on 82571 */ @@ -2322,53 +1673,26 @@ em_local_timer(void *arg) e1000_get_laa_state_82571(&adapter->hw)) e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); + if (adapter->hw.mac.type < em_mac_min) + lem_smartspeed(adapter); + /* Mask to use in the irq trigger */ - if (adapter->msix_mem) { - for (int i = 0; i < adapter->num_queues; i++, rxr++) - trigger |= rxr->ims; - rxr = adapter->rx_rings; - } else + if (adapter->intr_type == IFLIB_INTR_MSIX) { + for (i = 0, que = adapter->rx_queues; i < adapter->rx_num_queues; i++, que++) + trigger |= que->eims; + } else { trigger = E1000_ICS_RXDMT0; - - /* - ** Check on the state of the TX queue(s), this - ** can be done without the lock because its RO - ** and the HUNG state will be static if set. - */ - for (int i = 0; i < adapter->num_queues; i++, txr++) { - if (txr->busy == EM_TX_HUNG) - goto hung; - if (txr->busy >= EM_TX_MAXTRIES) - txr->busy = EM_TX_HUNG; - /* Schedule a TX tasklet if needed */ - if (txr->tx_avail <= EM_MAX_SCATTER) - taskqueue_enqueue(txr->tq, &txr->tx_task); } - - callout_reset(&adapter->timer, hz, em_local_timer, adapter); -#ifndef DEVICE_POLLING - /* Trigger an RX interrupt to guarantee mbuf refresh */ - E1000_WRITE_REG(&adapter->hw, E1000_ICS, trigger); -#endif - return; -hung: - /* Looks like we're hung */ - device_printf(adapter->dev, "Watchdog timeout Queue[%d]-- resetting\n", - txr->me); - em_print_debug_info(adapter); - if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); - adapter->watchdog_events++; - em_init_locked(adapter); } static void -em_update_link_status(struct adapter *adapter) +em_if_update_admin_status(if_ctx_t ctx) { + struct adapter *adapter = iflib_get_softc(ctx); struct e1000_hw *hw = &adapter->hw; - if_t ifp = adapter->ifp; - device_t dev = adapter->dev; - struct tx_ring *txr = adapter->tx_rings; + struct ifnet *ifp = iflib_get_ifp(ctx); + device_t dev = iflib_get_dev(ctx); u32 link_check = 0; /* Get the cached link value or read phy for real */ @@ -2382,13 +1706,14 @@ em_update_link_status(struct adapter *adapter) link_check = !hw->mac.get_link_status; if (link_check) /* ESB2 fix */ e1000_cfg_on_link_up(hw); - } else + } else { link_check = TRUE; + } break; case e1000_media_type_fiber: e1000_check_for_link(hw); link_check = (E1000_READ_REG(hw, E1000_STATUS) & - E1000_STATUS_LU); + E1000_STATUS_LU); break; case e1000_media_type_internal_serdes: e1000_check_for_link(hw); @@ -2403,18 +1728,6 @@ em_update_link_status(struct adapter *adapter) if (link_check && (adapter->link_active == 0)) { e1000_get_speed_and_duplex(hw, &adapter->link_speed, &adapter->link_duplex); - /* - ** There have proven to be problems with TSO when not - ** at full gigabit speed, so disable the assist automatically - ** when at lower speeds. -jfv - */ - if (adapter->link_speed != SPEED_1000) { - if_sethwassistbits(ifp, 0, CSUM_TSO); - if_setcapenablebit(ifp, 0, IFCAP_TSO4); - if_setcapabilitiesbit(ifp, 0, IFCAP_TSO4); - - } - /* Check if we must disable SPEED_MODE bit on PCI-E */ if ((adapter->link_speed != SPEED_1000) && ((hw->mac.type == e1000_82571) || @@ -2432,7 +1745,8 @@ em_update_link_status(struct adapter *adapter) adapter->link_active = 1; adapter->smartspeed = 0; if_setbaudrate(ifp, adapter->link_speed * 1000000); - if_link_state_change(ifp, LINK_STATE_UP); + iflib_link_state_change(ctx, LINK_STATE_UP, ifp->if_baudrate); + printf("Link state changed to up\n"); } else if (!link_check && (adapter->link_active == 1)) { if_setbaudrate(ifp, 0); adapter->link_speed = 0; @@ -2440,11 +1754,11 @@ em_update_link_status(struct adapter *adapter) if (bootverbose) device_printf(dev, "Link is Down\n"); adapter->link_active = 0; - /* Link down, disable hang detection */ - for (int i = 0; i < adapter->num_queues; i++, txr++) - txr->busy = EM_TX_IDLE; - if_link_state_change(ifp, LINK_STATE_DOWN); + iflib_link_state_change(ctx, LINK_STATE_DOWN, ifp->if_baudrate); + printf("link state changed to down\n"); } + + E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_LINK | E1000_IMS_LSC); } /********************************************************************* @@ -2457,35 +1771,15 @@ em_update_link_status(struct adapter *adapter) **********************************************************************/ static void -em_stop(void *arg) +em_if_stop(if_ctx_t ctx) { - struct adapter *adapter = arg; - if_t ifp = adapter->ifp; - struct tx_ring *txr = adapter->tx_rings; - - EM_CORE_LOCK_ASSERT(adapter); + struct adapter *adapter = iflib_get_softc(ctx); INIT_DEBUGOUT("em_stop: begin"); - em_disable_intr(adapter); - callout_stop(&adapter->timer); - - /* Tell the stack that the interface is no longer active */ - if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); - - /* Disarm Hang Detection. */ - for (int i = 0; i < adapter->num_queues; i++, txr++) { - EM_TX_LOCK(txr); - txr->busy = EM_TX_IDLE; - EM_TX_UNLOCK(txr); - } - - /* I219 needs some special flushing to avoid hangs */ - if (adapter->hw.mac.type == e1000_pch_spt) - em_flush_desc_rings(adapter); - e1000_reset_hw(&adapter->hw); - E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); + if (adapter->hw.mac.type >= e1000_82544) + E1000_WRITE_REG(&adapter->hw, E1000_WUFC, 0); e1000_led_off(&adapter->hw); e1000_cleanup_led(&adapter->hw); @@ -2498,12 +1792,12 @@ em_stop(void *arg) * **********************************************************************/ static void -em_identify_hardware(struct adapter *adapter) +em_identify_hardware(if_ctx_t ctx) { - device_t dev = adapter->dev; + device_t dev = iflib_get_dev(ctx); + struct adapter *adapter = iflib_get_softc(ctx); /* Make sure our PCI config space has the necessary stuff set */ - pci_enable_busmaster(dev); adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); /* Save off the information about this board */ @@ -2523,10 +1817,11 @@ em_identify_hardware(struct adapter *adapter) } static int -em_allocate_pci_resources(struct adapter *adapter) +em_allocate_pci_resources(if_ctx_t ctx) { - device_t dev = adapter->dev; - int rid; + struct adapter *adapter = iflib_get_softc(ctx); + device_t dev = iflib_get_dev(ctx); + int rid, val; rid = PCIR_BAR(0); adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, @@ -2535,498 +1830,398 @@ em_allocate_pci_resources(struct adapter *adapter) device_printf(dev, "Unable to allocate bus resource: memory\n"); return (ENXIO); } - adapter->osdep.mem_bus_space_tag = - rman_get_bustag(adapter->memory); + adapter->osdep.mem_bus_space_tag = rman_get_bustag(adapter->memory); adapter->osdep.mem_bus_space_handle = rman_get_bushandle(adapter->memory); adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; - adapter->hw.back = &adapter->osdep; - - return (0); -} - -/********************************************************************* - * - * Setup the Legacy or MSI Interrupt handler - * - **********************************************************************/ -int -em_allocate_legacy(struct adapter *adapter) -{ - device_t dev = adapter->dev; - struct tx_ring *txr = adapter->tx_rings; - int error, rid = 0; - - /* Manually turn off all interrupts */ - E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); - - if (adapter->msix == 1) /* using MSI */ - rid = 1; - /* We allocate a single interrupt resource */ - adapter->res = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); - if (adapter->res == NULL) { - device_printf(dev, "Unable to allocate bus resource: " - "interrupt\n"); - return (ENXIO); + /* Only older adapters use IO mapping */ + if (adapter->hw.mac.type < em_mac_min && + adapter->hw.mac.type > e1000_82543) { + /* Figure our where our IO BAR is ? */ + for (rid = PCIR_BAR(0); rid < PCIR_CIS;) { + val = pci_read_config(dev, rid, 4); + if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) { + adapter->io_rid = rid; + break; + } + rid += 4; + /* check for 64bit BAR */ + if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT) + rid += 4; + } + if (rid >= PCIR_CIS) { + device_printf(dev, "Unable to locate IO BAR\n"); + return (ENXIO); + } + adapter->ioport = bus_alloc_resource_any(dev, + SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE); + if (adapter->ioport == NULL) { + device_printf(dev, "Unable to allocate bus resource: " + "ioport\n"); + return (ENXIO); + } + adapter->hw.io_base = 0; + adapter->osdep.io_bus_space_tag = + rman_get_bustag(adapter->ioport); + adapter->osdep.io_bus_space_handle = + rman_get_bushandle(adapter->ioport); } - /* - * Allocate a fast interrupt and the associated - * deferred processing contexts. - */ - TASK_INIT(&adapter->que_task, 0, em_handle_que, adapter); - adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT, - taskqueue_thread_enqueue, &adapter->tq); - taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s que", - device_get_nameunit(adapter->dev)); - /* Use a TX only tasklet for local timer */ - TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr); - txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT, - taskqueue_thread_enqueue, &txr->tq); - taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq", - device_get_nameunit(adapter->dev)); - TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter); - if ((error = bus_setup_intr(dev, adapter->res, INTR_TYPE_NET, - em_irq_fast, NULL, adapter, &adapter->tag)) != 0) { - device_printf(dev, "Failed to register fast interrupt " - "handler: %d\n", error); - taskqueue_free(adapter->tq); - adapter->tq = NULL; - return (error); - } - + adapter->hw.back = &adapter->osdep; + return (0); } /********************************************************************* * * Setup the MSIX Interrupt handlers - * This is not really Multiqueue, rather - * its just separate interrupt vectors - * for TX, RX, and Link. * **********************************************************************/ -int -em_allocate_msix(struct adapter *adapter) +static int +em_if_msix_intr_assign(if_ctx_t ctx, int msix) { - device_t dev = adapter->dev; - struct tx_ring *txr = adapter->tx_rings; - struct rx_ring *rxr = adapter->rx_rings; - int error, rid, vector = 0; - int cpu_id = 0; - - - /* Make sure all interrupts are disabled */ - E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); + struct adapter *adapter = iflib_get_softc(ctx); + struct em_rx_queue *rx_que = adapter->rx_queues; + struct em_tx_queue *tx_que = adapter->tx_queues; + int error, rid, i, vector = 0, rx_vectors; + char buf[16]; /* First set up ring resources */ - for (int i = 0; i < adapter->num_queues; i++, rxr++, vector++) { - - /* RX ring */ + for (i = 0; i < adapter->rx_num_queues; i++, rx_que++, vector++) { rid = vector + 1; - - rxr->res = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &rid, RF_ACTIVE); - if (rxr->res == NULL) { - device_printf(dev, - "Unable to allocate bus resource: " - "RX MSIX Interrupt %d\n", i); - return (ENXIO); - } - if ((error = bus_setup_intr(dev, rxr->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_rx, - rxr, &rxr->tag)) != 0) { - device_printf(dev, "Failed to register RX handler"); - return (error); + snprintf(buf, sizeof(buf), "rxq%d", i); + error = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid, IFLIB_INTR_RXTX, em_msix_que, rx_que, rx_que->me, buf); + if (error) { + device_printf(iflib_get_dev(ctx), "Failed to allocate que int %d err: %d", i, error); + adapter->rx_num_queues = i + 1; + goto fail; } -#if __FreeBSD_version >= 800504 - bus_describe_intr(dev, rxr->res, rxr->tag, "rx%d", i); -#endif - rxr->msix = vector; - - if (em_last_bind_cpu < 0) - em_last_bind_cpu = CPU_FIRST(); - cpu_id = em_last_bind_cpu; - bus_bind_intr(dev, rxr->res, cpu_id); - - TASK_INIT(&rxr->rx_task, 0, em_handle_rx, rxr); - rxr->tq = taskqueue_create_fast("em_rxq", M_NOWAIT, - taskqueue_thread_enqueue, &rxr->tq); - taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq (cpuid %d)", - device_get_nameunit(adapter->dev), cpu_id); - /* - ** Set the bit to enable interrupt - ** in E1000_IMS -- bits 20 and 21 - ** are for RX0 and RX1, note this has - ** NOTHING to do with the MSIX vector - */ - rxr->ims = 1 << (20 + i); - adapter->ims |= rxr->ims; - adapter->ivars |= (8 | rxr->msix) << (i * 4); - em_last_bind_cpu = CPU_NEXT(em_last_bind_cpu); + rx_que->msix = vector; + + /* + * Set the bit to enable interrupt + * in E1000_IMS -- bits 20 and 21 + * are for RX0 and RX1, note this has + * NOTHING to do with the MSIX vector + */ + if (adapter->hw.mac.type == e1000_82574) { + rx_que->eims = 1 << (20 + i); + adapter->ims |= rx_que->eims; + adapter->ivars |= (8 | rx_que->msix) << (i * 4); + } else if (adapter->hw.mac.type == e1000_82575) + rx_que->eims = E1000_EICR_TX_QUEUE0 << vector; + else + rx_que->eims = 1 << vector; } + rx_vectors = vector; - for (int i = 0; i < adapter->num_queues; i++, txr++, vector++) { - /* TX ring */ + vector = 0; + for (i = 0; i < adapter->tx_num_queues; i++, tx_que++, vector++) { rid = vector + 1; - txr->res = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &rid, RF_ACTIVE); - if (txr->res == NULL) { - device_printf(dev, - "Unable to allocate bus resource: " - "TX MSIX Interrupt %d\n", i); - return (ENXIO); - } - if ((error = bus_setup_intr(dev, txr->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx, - txr, &txr->tag)) != 0) { - device_printf(dev, "Failed to register TX handler"); - return (error); - } -#if __FreeBSD_version >= 800504 - bus_describe_intr(dev, txr->res, txr->tag, "tx%d", i); -#endif - txr->msix = vector; - - if (em_last_bind_cpu < 0) - em_last_bind_cpu = CPU_FIRST(); - cpu_id = em_last_bind_cpu; - bus_bind_intr(dev, txr->res, cpu_id); - - TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr); - txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT, - taskqueue_thread_enqueue, &txr->tq); - taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq (cpuid %d)", - device_get_nameunit(adapter->dev), cpu_id); - /* - ** Set the bit to enable interrupt - ** in E1000_IMS -- bits 22 and 23 - ** are for TX0 and TX1, note this has - ** NOTHING to do with the MSIX vector - */ - txr->ims = 1 << (22 + i); - adapter->ims |= txr->ims; - adapter->ivars |= (8 | txr->msix) << (8 + (i * 4)); + snprintf(buf, sizeof(buf), "txq%d", i); + tx_que = &adapter->tx_queues[i]; + iflib_softirq_alloc_generic(ctx, rid, IFLIB_INTR_TX, tx_que, tx_que->me, buf); + + tx_que->msix = (vector % adapter->tx_num_queues); - em_last_bind_cpu = CPU_NEXT(em_last_bind_cpu); + /* + * Set the bit to enable interrupt + * in E1000_IMS -- bits 22 and 23 + * are for TX0 and TX1, note this has + * NOTHING to do with the MSIX vector + */ + if (adapter->hw.mac.type == e1000_82574) { + tx_que->eims = 1 << (22 + i); + adapter->ims |= tx_que->eims; + adapter->ivars |= (8 | tx_que->msix) << (8 + (i * 4)); + } else if (adapter->hw.mac.type == e1000_82575) { + tx_que->eims = E1000_EICR_TX_QUEUE0 << (i % adapter->tx_num_queues); + } else { + tx_que->eims = 1 << (i % adapter->tx_num_queues); + } } /* Link interrupt */ - rid = vector + 1; - adapter->res = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); - if (!adapter->res) { - device_printf(dev,"Unable to allocate " - "bus resource: Link interrupt [%d]\n", rid); - return (ENXIO); - } - /* Set the link handler function */ - error = bus_setup_intr(dev, adapter->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, - em_msix_link, adapter, &adapter->tag); + rid = rx_vectors + 1; + error = iflib_irq_alloc_generic(ctx, &adapter->irq, rid, IFLIB_INTR_ADMIN, em_msix_link, adapter, 0, "aq"); + if (error) { - adapter->res = NULL; - device_printf(dev, "Failed to register LINK handler"); - return (error); + device_printf(iflib_get_dev(ctx), "Failed to register admin handler"); + goto fail; + } + adapter->linkvec = rx_vectors; + if (adapter->hw.mac.type < igb_mac_min) { + adapter->ivars |= (8 | rx_vectors) << 16; + adapter->ivars |= 0x80000000; } -#if __FreeBSD_version >= 800504 - bus_describe_intr(dev, adapter->res, adapter->tag, "link"); -#endif - adapter->linkvec = vector; - adapter->ivars |= (8 | vector) << 16; - adapter->ivars |= 0x80000000; - return (0); +fail: + iflib_irq_free(ctx, &adapter->irq); + rx_que = adapter->rx_queues; + for (int i = 0; i < adapter->rx_num_queues; i++, rx_que++) + iflib_irq_free(ctx, &rx_que->que_irq); + return (error); } - static void -em_free_pci_resources(struct adapter *adapter) +igb_configure_queues(struct adapter *adapter) { - device_t dev = adapter->dev; - struct tx_ring *txr; - struct rx_ring *rxr; - int rid; + struct e1000_hw *hw = &adapter->hw; + struct em_rx_queue *rx_que; + struct em_tx_queue *tx_que; + u32 tmp, ivar = 0, newitr = 0; + /* First turn on RSS capability */ + if (adapter->hw.mac.type != e1000_82575) + E1000_WRITE_REG(hw, E1000_GPIE, + E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME | + E1000_GPIE_PBA | E1000_GPIE_NSICR); - /* - ** Release all the queue interrupt resources: - */ - for (int i = 0; i < adapter->num_queues; i++) { - txr = &adapter->tx_rings[i]; - /* an early abort? */ - if (txr == NULL) - break; - rid = txr->msix +1; - if (txr->tag != NULL) { - bus_teardown_intr(dev, txr->res, txr->tag); - txr->tag = NULL; + /* Turn on MSIX */ + switch (adapter->hw.mac.type) { + case e1000_82580: + case e1000_i350: + case e1000_i354: + case e1000_i210: + case e1000_i211: + case e1000_vfadapt: + case e1000_vfadapt_i350: + /* RX entries */ + for (int i = 0; i < adapter->rx_num_queues; i++) { + u32 index = i >> 1; + ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); + rx_que = &adapter->rx_queues[i]; + if (i & 1) { + ivar &= 0xFF00FFFF; + ivar |= (rx_que->msix | E1000_IVAR_VALID) << 16; + } else { + ivar &= 0xFFFFFF00; + ivar |= rx_que->msix | E1000_IVAR_VALID; + } + E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); + } + /* TX entries */ + for (int i = 0; i < adapter->tx_num_queues; i++) { + u32 index = i >> 1; + ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); + tx_que = &adapter->tx_queues[i]; + if (i & 1) { + ivar &= 0x00FFFFFF; + ivar |= (tx_que->msix | E1000_IVAR_VALID) << 24; + } else { + ivar &= 0xFFFF00FF; + ivar |= (tx_que->msix | E1000_IVAR_VALID) << 8; + } + E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); + adapter->que_mask |= tx_que->eims; } - if (txr->res != NULL) - bus_release_resource(dev, SYS_RES_IRQ, - rid, txr->res); - rxr = &adapter->rx_rings[i]; - /* an early abort? */ - if (rxr == NULL) - break; - rid = rxr->msix +1; - if (rxr->tag != NULL) { - bus_teardown_intr(dev, rxr->res, rxr->tag); - rxr->tag = NULL; + /* And for the link interrupt */ + ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8; + adapter->link_mask = 1 << adapter->linkvec; + E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); + break; + case e1000_82576: + /* RX entries */ + for (int i = 0; i < adapter->rx_num_queues; i++) { + u32 index = i & 0x7; /* Each IVAR has two entries */ + ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); + rx_que = &adapter->rx_queues[i]; + if (i < 8) { + ivar &= 0xFFFFFF00; + ivar |= rx_que->msix | E1000_IVAR_VALID; + } else { + ivar &= 0xFF00FFFF; + ivar |= (rx_que->msix | E1000_IVAR_VALID) << 16; + } + E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); + adapter->que_mask |= rx_que->eims; + } + /* TX entries */ + for (int i = 0; i < adapter->tx_num_queues; i++) { + u32 index = i & 0x7; /* Each IVAR has two entries */ + ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); + tx_que = &adapter->tx_queues[i]; + if (i < 8) { + ivar &= 0xFFFF00FF; + ivar |= (tx_que->msix | E1000_IVAR_VALID) << 8; + } else { + ivar &= 0x00FFFFFF; + ivar |= (tx_que->msix | E1000_IVAR_VALID) << 24; + } + E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); + adapter->que_mask |= tx_que->eims; } - if (rxr->res != NULL) - bus_release_resource(dev, SYS_RES_IRQ, - rid, rxr->res); + + /* And for the link interrupt */ + ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8; + adapter->link_mask = 1 << adapter->linkvec; + E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); + break; + + case e1000_82575: + /* enable MSI-X support*/ + tmp = E1000_READ_REG(hw, E1000_CTRL_EXT); + tmp |= E1000_CTRL_EXT_PBA_CLR; + /* Auto-Mask interrupts upon ICR read. */ + tmp |= E1000_CTRL_EXT_EIAME; + tmp |= E1000_CTRL_EXT_IRCA; + E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp); + + /* Queues */ + for (int i = 0; i < adapter->rx_num_queues; i++) { + rx_que = &adapter->rx_queues[i]; + tmp = E1000_EICR_RX_QUEUE0 << i; + tmp |= E1000_EICR_TX_QUEUE0 << i; + rx_que->eims = tmp; + E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), + i, rx_que->eims); + adapter->que_mask |= rx_que->eims; + } + + /* Link */ + E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec), + E1000_EIMS_OTHER); + adapter->link_mask |= E1000_EIMS_OTHER; + default: + break; } - if (adapter->linkvec) /* we are doing MSIX */ - rid = adapter->linkvec + 1; - else - (adapter->msix != 0) ? (rid = 1):(rid = 0); + /* Set the starting interrupt rate */ + if (em_max_interrupt_rate > 0) + newitr = (4000000 / em_max_interrupt_rate) & 0x7FFC; + + if (hw->mac.type == e1000_82575) + newitr |= newitr << 16; + else + newitr |= E1000_EITR_CNT_IGNR; - if (adapter->tag != NULL) { - bus_teardown_intr(dev, adapter->res, adapter->tag); - adapter->tag = NULL; + for (int i = 0; i < adapter->rx_num_queues; i++) { + rx_que = &adapter->rx_queues[i]; + E1000_WRITE_REG(hw, E1000_EITR(rx_que->msix), newitr); } - if (adapter->res != NULL) - bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res); + return; +} +static void +em_free_pci_resources(if_ctx_t ctx) +{ + struct adapter *adapter = iflib_get_softc(ctx); + struct em_rx_queue *que = adapter->rx_queues; + device_t dev = iflib_get_dev(ctx); - if (adapter->msix) - pci_release_msi(dev); + /* Release all msix queue resources */ + if (adapter->intr_type == IFLIB_INTR_MSIX) + iflib_irq_free(ctx, &adapter->irq); - if (adapter->msix_mem != NULL) - bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem); + for (int i = 0; i < adapter->rx_num_queues; i++, que++) { + iflib_irq_free(ctx, &que->que_irq); + } - if (adapter->memory != NULL) + /* First release all the interrupt resources */ + if (adapter->memory != NULL) { bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(0), adapter->memory); + PCIR_BAR(0), adapter->memory); + adapter->memory = NULL; + } - if (adapter->flash != NULL) + if (adapter->flash != NULL) { bus_release_resource(dev, SYS_RES_MEMORY, - EM_FLASH, adapter->flash); + EM_FLASH, adapter->flash); + adapter->flash = NULL; + } + if (adapter->ioport != NULL) + bus_release_resource(dev, SYS_RES_IOPORT, + adapter->io_rid, adapter->ioport); } -/* - * Setup MSI or MSI/X - */ +/* Setup MSI or MSI/X */ static int -em_setup_msix(struct adapter *adapter) +em_setup_msix(if_ctx_t ctx) { - device_t dev = adapter->dev; - int val; - - /* Nearly always going to use one queue */ - adapter->num_queues = 1; - - /* - ** Try using MSI-X for Hartwell adapters - */ - if ((adapter->hw.mac.type == e1000_82574) && - (em_enable_msix == TRUE)) { -#ifdef EM_MULTIQUEUE - adapter->num_queues = (em_num_queues == 1) ? 1 : 2; - if (adapter->num_queues > 1) - em_enable_vectors_82574(adapter); -#endif - /* Map the MSIX BAR */ - int rid = PCIR_BAR(EM_MSIX_BAR); - adapter->msix_mem = bus_alloc_resource_any(dev, - SYS_RES_MEMORY, &rid, RF_ACTIVE); - if (adapter->msix_mem == NULL) { - /* May not be enabled */ - device_printf(adapter->dev, - "Unable to map MSIX table \n"); - goto msi; - } - val = pci_msix_count(dev); - -#ifdef EM_MULTIQUEUE - /* We need 5 vectors in the multiqueue case */ - if (adapter->num_queues > 1 ) { - if (val >= 5) - val = 5; - else { - adapter->num_queues = 1; - device_printf(adapter->dev, - "Insufficient MSIX vectors for >1 queue, " - "using single queue...\n"); - goto msix_one; - } - } else { -msix_one: -#endif - if (val >= 3) - val = 3; - else { - device_printf(adapter->dev, - "Insufficient MSIX vectors, using MSI\n"); - goto msi; - } -#ifdef EM_MULTIQUEUE - } -#endif + struct adapter *adapter = iflib_get_softc(ctx); - if ((pci_alloc_msix(dev, &val) == 0)) { - device_printf(adapter->dev, - "Using MSIX interrupts " - "with %d vectors\n", val); - return (val); - } - - /* - ** If MSIX alloc failed or provided us with - ** less than needed, free and fall through to MSI - */ - pci_release_msi(dev); - } -msi: - if (adapter->msix_mem != NULL) { - bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem); - adapter->msix_mem = NULL; + if (adapter->hw.mac.type == e1000_82574) { + em_enable_vectors_82574(ctx); } - val = 1; - if (pci_alloc_msi(dev, &val) == 0) { - device_printf(adapter->dev, "Using an MSI interrupt\n"); - return (val); - } - /* Should only happen due to manual configuration */ - device_printf(adapter->dev,"No MSI/MSIX using a Legacy IRQ\n"); return (0); } +/********************************************************************* + * + * Initialize the hardware to a configuration + * as specified by the adapter structure. + * + **********************************************************************/ -/* -** The 3 following flush routines are used as a workaround in the -** I219 client parts and only for them. -** -** em_flush_tx_ring - remove all descriptors from the tx_ring -** -** We want to clear all pending descriptors from the TX ring. -** zeroing happens when the HW reads the regs. We assign the ring itself as -** the data of the next descriptor. We don't care about the data we are about -** to reset the HW. -*/ -static void -em_flush_tx_ring(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - struct tx_ring *txr = adapter->tx_rings; - struct e1000_tx_desc *txd; - u32 tctl, txd_lower = E1000_TXD_CMD_IFCS; - u16 size = 512; - - tctl = E1000_READ_REG(hw, E1000_TCTL); - E1000_WRITE_REG(hw, E1000_TCTL, tctl | E1000_TCTL_EN); - - txd = &txr->tx_base[txr->next_avail_desc++]; - if (txr->next_avail_desc == adapter->num_tx_desc) - txr->next_avail_desc = 0; - - /* Just use the ring as a dummy buffer addr */ - txd->buffer_addr = txr->txdma.dma_paddr; - txd->lower.data = htole32(txd_lower | size); - txd->upper.data = 0; - - /* flush descriptors to memory before notifying the HW */ - wmb(); - - E1000_WRITE_REG(hw, E1000_TDT(0), txr->next_avail_desc); - mb(); - usec_delay(250); -} - -/* -** em_flush_rx_ring - remove all descriptors from the rx_ring -** -** Mark all descriptors in the RX ring as consumed and disable the rx ring -*/ static void -em_flush_rx_ring(struct adapter *adapter) +lem_smartspeed(struct adapter *adapter) { - struct e1000_hw *hw = &adapter->hw; - u32 rctl, rxdctl; - - rctl = E1000_READ_REG(hw, E1000_RCTL); - E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); - E1000_WRITE_FLUSH(hw); - usec_delay(150); - - rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0)); - /* zero the lower 14 bits (prefetch and host thresholds) */ - rxdctl &= 0xffffc000; - /* - * update thresholds: prefetch threshold to 31, host threshold to 1 - * and make sure the granularity is "descriptors" and not "cache lines" - */ - rxdctl |= (0x1F | (1 << 8) | E1000_RXDCTL_THRESH_UNIT_DESC); - E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl); - - /* momentarily enable the RX ring for the changes to take effect */ - E1000_WRITE_REG(hw, E1000_RCTL, rctl | E1000_RCTL_EN); - E1000_WRITE_FLUSH(hw); - usec_delay(150); - E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); -} + u16 phy_tmp; -/* -** em_flush_desc_rings - remove all descriptors from the descriptor rings -** -** In i219, the descriptor rings must be emptied before resetting the HW -** or before changing the device state to D3 during runtime (runtime PM). -** -** Failure to do this will cause the HW to enter a unit hang state which can -** only be released by PCI reset on the device -** -*/ -static void -em_flush_desc_rings(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - device_t dev = adapter->dev; - u16 hang_state; - u32 fext_nvm11, tdlen; - - /* First, disable MULR fix in FEXTNVM11 */ - fext_nvm11 = E1000_READ_REG(hw, E1000_FEXTNVM11); - fext_nvm11 |= E1000_FEXTNVM11_DISABLE_MULR_FIX; - E1000_WRITE_REG(hw, E1000_FEXTNVM11, fext_nvm11); - - /* do nothing if we're not in faulty state, or if the queue is empty */ - tdlen = E1000_READ_REG(hw, E1000_TDLEN(0)); - hang_state = pci_read_config(dev, PCICFG_DESC_RING_STATUS, 2); - if (!(hang_state & FLUSH_DESC_REQUIRED) || !tdlen) + if (adapter->link_active || (adapter->hw.phy.type != e1000_phy_igp) || + adapter->hw.mac.autoneg == 0 || + (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0) return; - em_flush_tx_ring(adapter); - /* recheck, maybe the fault is caused by the rx ring */ - hang_state = pci_read_config(dev, PCICFG_DESC_RING_STATUS, 2); - if (hang_state & FLUSH_DESC_REQUIRED) - em_flush_rx_ring(adapter); + if (adapter->smartspeed == 0) { + /* If Master/Slave config fault is asserted twice, + * we assume back-to-back */ + e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp); + if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT)) + return; + e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp); + if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) { + e1000_read_phy_reg(&adapter->hw, + PHY_1000T_CTRL, &phy_tmp); + if(phy_tmp & CR_1000T_MS_ENABLE) { + phy_tmp &= ~CR_1000T_MS_ENABLE; + e1000_write_phy_reg(&adapter->hw, + PHY_1000T_CTRL, phy_tmp); + adapter->smartspeed++; + if(adapter->hw.mac.autoneg && + !e1000_copper_link_autoneg(&adapter->hw) && + !e1000_read_phy_reg(&adapter->hw, + PHY_CONTROL, &phy_tmp)) { + phy_tmp |= (MII_CR_AUTO_NEG_EN | + MII_CR_RESTART_AUTO_NEG); + e1000_write_phy_reg(&adapter->hw, + PHY_CONTROL, phy_tmp); + } + } + } + return; + } else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) { + /* If still no link, perhaps using 2/3 pair cable */ + e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp); + phy_tmp |= CR_1000T_MS_ENABLE; + e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp); + if(adapter->hw.mac.autoneg && + !e1000_copper_link_autoneg(&adapter->hw) && + !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) { + phy_tmp |= (MII_CR_AUTO_NEG_EN | + MII_CR_RESTART_AUTO_NEG); + e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp); + } + } + /* Restart process after EM_SMARTSPEED_MAX iterations */ + if(adapter->smartspeed++ == EM_SMARTSPEED_MAX) + adapter->smartspeed = 0; } -/********************************************************************* - * - * Initialize the hardware to a configuration - * as specified by the adapter structure. - * - **********************************************************************/ static void -em_reset(struct adapter *adapter) +em_reset(if_ctx_t ctx) { - device_t dev = adapter->dev; - if_t ifp = adapter->ifp; - struct e1000_hw *hw = &adapter->hw; - u16 rx_buffer_size; - u32 pba; + device_t dev = iflib_get_dev(ctx); + struct adapter *adapter = iflib_get_softc(ctx); + struct ifnet *ifp = iflib_get_ifp(ctx); + struct e1000_hw *hw = &adapter->hw; + u16 rx_buffer_size; + u32 pba; INIT_DEBUGOUT("em_reset: begin"); @@ -3077,13 +2272,62 @@ em_reset(struct adapter *adapter) case e1000_pch_spt: pba = E1000_PBA_26K; break; + case e1000_82575: + pba = E1000_PBA_32K; + break; + case e1000_82576: + case e1000_vfadapt: + pba = E1000_READ_REG(hw, E1000_RXPBS); + pba &= E1000_RXPBS_SIZE_MASK_82576; + break; + case e1000_82580: + case e1000_i350: + case e1000_i354: + case e1000_vfadapt_i350: + pba = E1000_READ_REG(hw, E1000_RXPBS); + pba = e1000_rxpbs_adjust_82580(pba); + break; + case e1000_i210: + case e1000_i211: + pba = E1000_PBA_34K; + break; default: if (adapter->hw.mac.max_frame_size > 8192) pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */ else pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */ } - E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba); + + /* Special needs in case of Jumbo frames */ + if ((hw->mac.type == e1000_82575) && (ifp->if_mtu > ETHERMTU)) { + u32 tx_space, min_tx, min_rx; + pba = E1000_READ_REG(hw, E1000_PBA); + tx_space = pba >> 16; + pba &= 0xffff; + min_tx = (adapter->hw.mac.max_frame_size + + sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2; + min_tx = roundup2(min_tx, 1024); + min_tx >>= 10; + min_rx = adapter->hw.mac.max_frame_size; + min_rx = roundup2(min_rx, 1024); + min_rx >>= 10; + if (tx_space < min_tx && + ((min_tx - tx_space) < pba)) { + pba = pba - (min_tx - tx_space); + /* + * if short on rx space, rx wins + * and must trump tx adjustment + */ + if (pba < min_rx) + pba = min_rx; + } + E1000_WRITE_REG(hw, E1000_PBA, pba); + } + + if (hw->mac.type < igb_mac_min) + E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba); + + INIT_DEBUGOUT1("em_reset: pba=%dK",pba); /* * These parameters control the automatic generation (Tx) and @@ -3099,7 +2343,7 @@ em_reset(struct adapter *adapter) * by 1500. * - The pause time is fairly large at 1000 x 512ns = 512 usec. */ - rx_buffer_size = ((E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10 ); + rx_buffer_size = (pba & 0xffff) << 10; hw->fc.high_water = rx_buffer_size - roundup2(adapter->hw.mac.max_frame_size, 1024); hw->fc.low_water = hw->fc.high_water - 1500; @@ -3120,7 +2364,7 @@ em_reset(struct adapter *adapter) switch (hw->mac.type) { case e1000_pchlan: /* Workaround: no TX flow ctrl for PCH */ - hw->fc.requested_mode = e1000_fc_rx_pause; + hw->fc.requested_mode = e1000_fc_rx_pause; hw->fc.pause_time = 0xFFFF; /* override */ if (if_getmtu(ifp) > ETHERMTU) { hw->fc.high_water = 0x3500; @@ -3144,13 +2388,28 @@ em_reset(struct adapter *adapter) else E1000_WRITE_REG(hw, E1000_PBA, 26); break; + case e1000_82575: + case e1000_82576: + /* 8-byte granularity */ + hw->fc.low_water = hw->fc.high_water - 8; + break; + case e1000_82580: + case e1000_i350: + case e1000_i354: + case e1000_i210: + case e1000_i211: + case e1000_vfadapt: + case e1000_vfadapt_i350: + /* 16-byte granularity */ + hw->fc.low_water = hw->fc.high_water - 16; + break; case e1000_ich9lan: case e1000_ich10lan: if (if_getmtu(ifp) > ETHERMTU) { hw->fc.high_water = 0x2800; hw->fc.low_water = hw->fc.high_water - 8; break; - } + } /* else fall thru */ default: if (hw->mac.type == e1000_80003es2lan) @@ -3158,13 +2417,9 @@ em_reset(struct adapter *adapter) break; } - /* I219 needs some special flushing to avoid hangs */ - if (hw->mac.type == e1000_pch_spt) - em_flush_desc_rings(adapter); - /* Issue a global reset */ e1000_reset_hw(hw); - E1000_WRITE_REG(hw, E1000_WUC, 0); + E1000_WRITE_REG(hw, E1000_WUFC, 0); em_disable_aspm(adapter); /* and a re-init */ if (e1000_init_hw(hw) < 0) { @@ -3175,7 +2430,145 @@ em_reset(struct adapter *adapter) E1000_WRITE_REG(hw, E1000_VET, ETHERTYPE_VLAN); e1000_get_phy_info(hw); e1000_check_for_link(hw); - return; +} + +#define RSSKEYLEN 10 +static void +em_initialize_rss_mapping(struct adapter *adapter) +{ + uint8_t rss_key[4 * RSSKEYLEN]; + uint32_t reta = 0; + struct e1000_hw *hw = &adapter->hw; + int i; + + /* + * Configure RSS key + */ + arc4rand(rss_key, sizeof(rss_key), 0); + for (i = 0; i < RSSKEYLEN; ++i) { + uint32_t rssrk = 0; + + rssrk = EM_RSSRK_VAL(rss_key, i); + E1000_WRITE_REG(hw,E1000_RSSRK(i), rssrk); + } + + /* + * Configure RSS redirect table in following fashion: + * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)] + */ + for (i = 0; i < sizeof(reta); ++i) { + uint32_t q; + + q = (i % adapter->rx_num_queues) << 7; + reta |= q << (8 * i); + } + + for (i = 0; i < 32; ++i) + E1000_WRITE_REG(hw, E1000_RETA(i), reta); + + E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_RSS_ENABLE_2Q | + E1000_MRQC_RSS_FIELD_IPV4_TCP | + E1000_MRQC_RSS_FIELD_IPV4 | + E1000_MRQC_RSS_FIELD_IPV6_TCP_EX | + E1000_MRQC_RSS_FIELD_IPV6_EX | + E1000_MRQC_RSS_FIELD_IPV6); + +} + +static void +igb_initialize_rss_mapping(struct adapter *adapter) +{ + struct e1000_hw *hw = &adapter->hw; + int i; + int queue_id; + u32 reta; + u32 rss_key[10], mrqc, shift = 0; + + /* XXX? */ + if (adapter->hw.mac.type == e1000_82575) + shift = 6; + + /* + * The redirection table controls which destination + * queue each bucket redirects traffic to. + * Each DWORD represents four queues, with the LSB + * being the first queue in the DWORD. + * + * This just allocates buckets to queues using round-robin + * allocation. + * + * NOTE: It Just Happens to line up with the default + * RSS allocation method. + */ + + /* Warning FM follows */ + reta = 0; + for (i = 0; i < 128; i++) { +#ifdef RSS + queue_id = rss_get_indirection_to_bucket(i); + /* + * If we have more queues than buckets, we'll + * end up mapping buckets to a subset of the + * queues. + * + * If we have more buckets than queues, we'll + * end up instead assigning multiple buckets + * to queues. + * + * Both are suboptimal, but we need to handle + * the case so we don't go out of bounds + * indexing arrays and such. + */ + queue_id = queue_id % adapter->rx_num_queues; +#else + queue_id = (i % adapter->rx_num_queues); +#endif + /* Adjust if required */ + queue_id = queue_id << shift; + + /* + * The low 8 bits are for hash value (n+0); + * The next 8 bits are for hash value (n+1), etc. + */ + reta = reta >> 8; + reta = reta | ( ((uint32_t) queue_id) << 24); + if ((i & 3) == 3) { + E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta); + reta = 0; + } + } + + /* Now fill in hash table */ + + /* + * MRQC: Multiple Receive Queues Command + * Set queuing to RSS control, number depends on the device. + */ + mrqc = E1000_MRQC_ENABLE_RSS_8Q; + +#ifdef RSS + /* XXX ew typecasting */ + rss_getkey((uint8_t *) &rss_key); +#else + arc4rand(&rss_key, sizeof(rss_key), 0); +#endif + for (i = 0; i < 10; i++) + E1000_WRITE_REG_ARRAY(hw, + E1000_RSSRK(0), i, rss_key[i]); + + /* + * Configure the RSS fields to hash upon. + */ + mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | + E1000_MRQC_RSS_FIELD_IPV4_TCP); + mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 | + E1000_MRQC_RSS_FIELD_IPV6_TCP); + mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP | + E1000_MRQC_RSS_FIELD_IPV6_UDP); + mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | + E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); + + E1000_WRITE_REG(hw, E1000_MRQC, mrqc); } /********************************************************************* @@ -3184,497 +2577,239 @@ em_reset(struct adapter *adapter) * **********************************************************************/ static int -em_setup_interface(device_t dev, struct adapter *adapter) +em_setup_interface(if_ctx_t ctx) { - if_t ifp; + struct ifnet *ifp = iflib_get_ifp(ctx); + struct adapter *adapter = iflib_get_softc(ctx); + if_softc_ctx_t scctx = adapter->shared; + uint64_t cap = 0; INIT_DEBUGOUT("em_setup_interface: begin"); - ifp = adapter->ifp = if_gethandle(IFT_ETHER); - if (ifp == 0) { - device_printf(dev, "can not allocate ifnet structure\n"); - return (-1); - } - if_initname(ifp, device_get_name(dev), device_get_unit(dev)); - if_setdev(ifp, dev); - if_setinitfn(ifp, em_init); - if_setsoftc(ifp, adapter); - if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); - if_setioctlfn(ifp, em_ioctl); - if_setgetcounterfn(ifp, em_get_counter); - /* TSO parameters */ - ifp->if_hw_tsomax = IP_MAXPACKET; + if_sethwtsomax(ifp, IP_MAXPACKET); /* Take m_pullup(9)'s in em_xmit() w/ TSO into acount. */ - ifp->if_hw_tsomaxsegcount = EM_MAX_SCATTER - 5; - ifp->if_hw_tsomaxsegsize = EM_TSO_SEG_SIZE; - -#ifdef EM_MULTIQUEUE - /* Multiqueue stack interface */ - if_settransmitfn(ifp, em_mq_start); - if_setqflushfn(ifp, em_qflush); -#else - if_setstartfn(ifp, em_start); - if_setsendqlen(ifp, adapter->num_tx_desc - 1); - if_setsendqready(ifp); -#endif - - ether_ifattach(ifp, adapter->hw.mac.addr); + if_sethwtsomaxsegcount(ifp, EM_MAX_SCATTER - 5); + if_sethwtsomaxsegsize(ifp, EM_TSO_SEG_SIZE); - if_setcapabilities(ifp, 0); - if_setcapenable(ifp, 0); + /* Single Queue */ + if (adapter->tx_num_queues == 1) { + if_setsendqlen(ifp, scctx->isc_ntxd[0] - 1); + if_setsendqready(ifp); + } + cap = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO4; + cap |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWTSO | IFCAP_VLAN_MTU; - if_setcapabilitiesbit(ifp, IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | - IFCAP_TSO4, 0); /* * Tell the upper layer(s) we * support full VLAN capability */ if_setifheaderlen(ifp, sizeof(struct ether_vlan_header)); - if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWTSO | - IFCAP_VLAN_MTU, 0); - if_setcapenable(ifp, if_getcapabilities(ifp)); + if_setcapabilitiesbit(ifp, cap, 0); /* - ** Don't turn this on by default, if vlans are - ** created on another pseudo device (eg. lagg) - ** then vlan events are not passed thru, breaking - ** operation, but with HW FILTER off it works. If - ** using vlans directly on the em driver you can - ** enable this and get full hardware tag filtering. - */ + * Don't turn this on by default, if vlans are + * created on another pseudo device (eg. lagg) + * then vlan events are not passed thru, breaking + * operation, but with HW FILTER off it works. If + * using vlans directly on the em driver you can + * enable this and get full hardware tag filtering. + */ if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWFILTER,0); -#ifdef DEVICE_POLLING - if_setcapabilitiesbit(ifp, IFCAP_POLLING,0); -#endif - /* Enable only WOL MAGIC by default */ if (adapter->wol) { - if_setcapabilitiesbit(ifp, IFCAP_WOL, 0); - if_setcapenablebit(ifp, IFCAP_WOL_MAGIC, 0); + if_setcapenablebit(ifp, IFCAP_WOL_MAGIC, + IFCAP_WOL_MCAST| IFCAP_WOL_UCAST); + } else { + if_setcapenablebit(ifp, 0, IFCAP_WOL_MAGIC | + IFCAP_WOL_MCAST| IFCAP_WOL_UCAST); } - + /* * Specify the media types supported by this adapter and register * callbacks to update media and link information */ - ifmedia_init(&adapter->media, IFM_IMASK, - em_media_change, em_media_status); if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { u_char fiber_type = IFM_1000_SX; /* default type */ - ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX, - 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL); + if (adapter->hw.mac.type == e1000_82545) + fiber_type = IFM_1000_LX; + ifmedia_add(adapter->media, IFM_ETHER | fiber_type | IFM_FDX, 0, NULL); + ifmedia_add(adapter->media, IFM_ETHER | fiber_type, 0, NULL); } else { - ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, - 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, - 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, - 0, NULL); + ifmedia_add(adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); + ifmedia_add(adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); + ifmedia_add(adapter->media, IFM_ETHER | IFM_100_TX, 0, NULL); + ifmedia_add(adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); if (adapter->hw.phy.type != e1000_phy_ife) { - ifmedia_add(&adapter->media, - IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); - ifmedia_add(&adapter->media, - IFM_ETHER | IFM_1000_T, 0, NULL); + ifmedia_add(adapter->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); + ifmedia_add(adapter->media, IFM_ETHER | IFM_1000_T, 0, NULL); } } - ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); - ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); + ifmedia_add(adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); + ifmedia_set(adapter->media, IFM_ETHER | IFM_AUTO); return (0); } - -/* - * Manage DMA'able memory. - */ -static void -em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) -{ - if (error) - return; - *(bus_addr_t *) arg = segs[0].ds_addr; -} - static int -em_dma_malloc(struct adapter *adapter, bus_size_t size, - struct em_dma_alloc *dma, int mapflags) +em_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets) { - int error; - - error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ - EM_DBA_ALIGN, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - size, /* maxsize */ - 1, /* nsegments */ - size, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockarg */ - &dma->dma_tag); - if (error) { - device_printf(adapter->dev, - "%s: bus_dma_tag_create failed: %d\n", - __func__, error); - goto fail_0; - } + struct adapter *adapter = iflib_get_softc(ctx); + if_softc_ctx_t scctx = adapter->shared; + int error = E1000_SUCCESS; + struct em_tx_queue *que; + int i, j; - error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, - BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map); - if (error) { - device_printf(adapter->dev, - "%s: bus_dmamem_alloc(%ju) failed: %d\n", - __func__, (uintmax_t)size, error); - goto fail_2; - } - - dma->dma_paddr = 0; - error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, - size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); - if (error || dma->dma_paddr == 0) { - device_printf(adapter->dev, - "%s: bus_dmamap_load failed: %d\n", - __func__, error); - goto fail_3; - } - - return (0); - -fail_3: - bus_dmamap_unload(dma->dma_tag, dma->dma_map); -fail_2: - bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); - bus_dma_tag_destroy(dma->dma_tag); -fail_0: - dma->dma_tag = NULL; - - return (error); -} - -static void -em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma) -{ - if (dma->dma_tag == NULL) - return; - if (dma->dma_paddr != 0) { - bus_dmamap_sync(dma->dma_tag, dma->dma_map, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(dma->dma_tag, dma->dma_map); - dma->dma_paddr = 0; - } - if (dma->dma_vaddr != NULL) { - bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); - dma->dma_vaddr = NULL; - } - bus_dma_tag_destroy(dma->dma_tag); - dma->dma_tag = NULL; -} - - -/********************************************************************* - * - * Allocate memory for the transmit and receive rings, and then - * the descriptors associated with each, called only once at attach. - * - **********************************************************************/ -static int -em_allocate_queues(struct adapter *adapter) -{ - device_t dev = adapter->dev; - struct tx_ring *txr = NULL; - struct rx_ring *rxr = NULL; - int rsize, tsize, error = E1000_SUCCESS; - int txconf = 0, rxconf = 0; - - - /* Allocate the TX ring struct memory */ - if (!(adapter->tx_rings = - (struct tx_ring *) malloc(sizeof(struct tx_ring) * - adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate TX ring memory\n"); - error = ENOMEM; - goto fail; - } + MPASS(adapter->tx_num_queues > 0); + MPASS(adapter->tx_num_queues == ntxqsets); - /* Now allocate the RX */ - if (!(adapter->rx_rings = - (struct rx_ring *) malloc(sizeof(struct rx_ring) * - adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate RX ring memory\n"); - error = ENOMEM; - goto rx_fail; + /* First allocate the top level queue structs */ + if (!(adapter->tx_queues = + (struct em_tx_queue *) malloc(sizeof(struct em_tx_queue) * + adapter->tx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { + device_printf(iflib_get_dev(ctx), "Unable to allocate queue memory\n"); + return(ENOMEM); } - tsize = roundup2(adapter->num_tx_desc * - sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); - /* - * Now set up the TX queues, txconf is needed to handle the - * possibility that things fail midcourse and we need to - * undo memory gracefully - */ - for (int i = 0; i < adapter->num_queues; i++, txconf++) { + for (i = 0, que = adapter->tx_queues; i < adapter->tx_num_queues; i++, que++) { /* Set up some basics */ - txr = &adapter->tx_rings[i]; - txr->adapter = adapter; - txr->me = i; - - /* Initialize the TX lock */ - snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", - device_get_nameunit(dev), txr->me); - mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); - - if (em_dma_malloc(adapter, tsize, - &txr->txdma, BUS_DMA_NOWAIT)) { - device_printf(dev, - "Unable to allocate TX Descriptor memory\n"); - error = ENOMEM; - goto err_tx_desc; - } - txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr; - bzero((void *)txr->tx_base, tsize); - - if (em_allocate_transmit_buffers(txr)) { - device_printf(dev, - "Critical Failure setting up transmit buffers\n"); - error = ENOMEM; - goto err_tx_desc; - } -#if __FreeBSD_version >= 800000 - /* Allocate a buf ring */ - txr->br = buf_ring_alloc(4096, M_DEVBUF, - M_WAITOK, &txr->tx_mtx); -#endif - } - /* - * Next the RX queues... - */ - rsize = roundup2(adapter->num_rx_desc * - sizeof(union e1000_rx_desc_extended), EM_DBA_ALIGN); - for (int i = 0; i < adapter->num_queues; i++, rxconf++) { - rxr = &adapter->rx_rings[i]; - rxr->adapter = adapter; - rxr->me = i; - - /* Initialize the RX lock */ - snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", - device_get_nameunit(dev), txr->me); - mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); - - if (em_dma_malloc(adapter, rsize, - &rxr->rxdma, BUS_DMA_NOWAIT)) { - device_printf(dev, - "Unable to allocate RxDescriptor memory\n"); - error = ENOMEM; - goto err_rx_desc; - } - rxr->rx_base = (union e1000_rx_desc_extended *)rxr->rxdma.dma_vaddr; - bzero((void *)rxr->rx_base, rsize); + struct tx_ring *txr = &que->txr; + txr->adapter = que->adapter = adapter; + que->me = txr->me = i; - /* Allocate receive buffers for the ring*/ - if (em_allocate_receive_buffers(rxr)) { - device_printf(dev, - "Critical Failure setting up receive buffers\n"); + /* Allocate report status array */ + if (!(txr->tx_rsq = (qidx_t *) malloc(sizeof(qidx_t) * scctx->isc_ntxd[0], M_DEVBUF, M_NOWAIT | M_ZERO))) { + device_printf(iflib_get_dev(ctx), "failed to allocate rs_idxs memory\n"); error = ENOMEM; - goto err_rx_desc; + goto fail; } + for (j = 0; j < scctx->isc_ntxd[0]; j++) + txr->tx_rsq[j] = QIDX_INVALID; + /* get the virtual and physical address of the hardware queues */ + txr->tx_base = (struct e1000_tx_desc *)vaddrs[i*ntxqs]; + txr->tx_paddr = paddrs[i*ntxqs]; } - + + device_printf(iflib_get_dev(ctx), "allocated for %d tx_queues\n", adapter->tx_num_queues); return (0); - -err_rx_desc: - for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) - em_dma_free(adapter, &rxr->rxdma); -err_tx_desc: - for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) - em_dma_free(adapter, &txr->txdma); - free(adapter->rx_rings, M_DEVBUF); -rx_fail: -#if __FreeBSD_version >= 800000 - buf_ring_free(txr->br, M_DEVBUF); -#endif - free(adapter->tx_rings, M_DEVBUF); fail: + em_if_queues_free(ctx); return (error); } - -/********************************************************************* - * - * Allocate memory for tx_buffer structures. The tx_buffer stores all - * the information needed to transmit a packet on the wire. This is - * called only once at attach, setup is done every reset. - * - **********************************************************************/ static int -em_allocate_transmit_buffers(struct tx_ring *txr) +em_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nrxqs, int nrxqsets) { - struct adapter *adapter = txr->adapter; - device_t dev = adapter->dev; - struct em_txbuffer *txbuf; - int error, i; + struct adapter *adapter = iflib_get_softc(ctx); + int error = E1000_SUCCESS; + struct em_rx_queue *que; + int i; - /* - * Setup DMA descriptor areas. - */ - if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - EM_TSO_SIZE, /* maxsize */ - EM_MAX_SCATTER, /* nsegments */ - PAGE_SIZE, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockfuncarg */ - &txr->txtag))) { - device_printf(dev,"Unable to allocate TX DMA tag\n"); - goto fail; - } + MPASS(adapter->rx_num_queues > 0); + MPASS(adapter->rx_num_queues == nrxqsets); - if (!(txr->tx_buffers = - (struct em_txbuffer *) malloc(sizeof(struct em_txbuffer) * - adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate tx_buffer memory\n"); + /* First allocate the top level queue structs */ + if (!(adapter->rx_queues = + (struct em_rx_queue *) malloc(sizeof(struct em_rx_queue) * + adapter->rx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { + device_printf(iflib_get_dev(ctx), "Unable to allocate queue memory\n"); error = ENOMEM; - goto fail; + goto fail; } - /* Create the descriptor buffer dma maps */ - txbuf = txr->tx_buffers; - for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { - error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); - if (error != 0) { - device_printf(dev, "Unable to create TX DMA map\n"); - goto fail; - } + for (i = 0, que = adapter->rx_queues; i < nrxqsets; i++, que++) { + /* Set up some basics */ + struct rx_ring *rxr = &que->rxr; + rxr->adapter = que->adapter = adapter; + rxr->que = que; + que->me = rxr->me = i; + + /* get the virtual and physical address of the hardware queues */ + rxr->rx_base = (union e1000_rx_desc_extended *)vaddrs[i*nrxqs]; + rxr->rx_paddr = paddrs[i*nrxqs]; } - return 0; + device_printf(iflib_get_dev(ctx), "allocated for %d rx_queues\n", adapter->rx_num_queues); + + return (0); fail: - /* We free all, it handles case where we are in the middle */ - em_free_transmit_structures(adapter); + em_if_queues_free(ctx); return (error); } -/********************************************************************* - * - * Initialize a transmit ring. - * - **********************************************************************/ static void -em_setup_transmit_ring(struct tx_ring *txr) +em_if_queues_free(if_ctx_t ctx) { - struct adapter *adapter = txr->adapter; - struct em_txbuffer *txbuf; - int i; -#ifdef DEV_NETMAP - struct netmap_slot *slot; - struct netmap_adapter *na = netmap_getna(adapter->ifp); -#endif /* DEV_NETMAP */ - - /* Clear the old descriptor contents */ - EM_TX_LOCK(txr); -#ifdef DEV_NETMAP - slot = netmap_reset(na, NR_TX, txr->me, 0); -#endif /* DEV_NETMAP */ - - bzero((void *)txr->tx_base, - (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc); - /* Reset indices */ - txr->next_avail_desc = 0; - txr->next_to_clean = 0; - - /* Free any existing tx buffers. */ - txbuf = txr->tx_buffers; - for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { - if (txbuf->m_head != NULL) { - bus_dmamap_sync(txr->txtag, txbuf->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(txr->txtag, txbuf->map); - m_freem(txbuf->m_head); - txbuf->m_head = NULL; - } -#ifdef DEV_NETMAP - if (slot) { - int si = netmap_idx_n2k(&na->tx_rings[txr->me], i); - uint64_t paddr; - void *addr; - - addr = PNMB(na, slot + si, &paddr); - txr->tx_base[i].buffer_addr = htole64(paddr); - /* reload the map for netmap mode */ - netmap_load_map(na, txr->txtag, txbuf->map, addr); - } -#endif /* DEV_NETMAP */ + struct adapter *adapter = iflib_get_softc(ctx); + struct em_tx_queue *tx_que = adapter->tx_queues; + struct em_rx_queue *rx_que = adapter->rx_queues; + + if (tx_que != NULL) { + for (int i = 0; i < adapter->tx_num_queues; i++, tx_que++) { + struct tx_ring *txr = &tx_que->txr; + if (txr->tx_rsq == NULL) + break; - /* clear the watch index */ - txbuf->next_eop = -1; - } + free(txr->tx_rsq, M_DEVBUF); + txr->tx_rsq = NULL; + } + free(adapter->tx_queues, M_DEVBUF); + adapter->tx_queues = NULL; + } - /* Set number of descriptors available */ - txr->tx_avail = adapter->num_tx_desc; - txr->busy = EM_TX_IDLE; + if (rx_que != NULL) { + free(adapter->rx_queues, M_DEVBUF); + adapter->rx_queues = NULL; + } - /* Clear checksum offload context. */ - txr->last_hw_offload = 0; - txr->last_hw_ipcss = 0; - txr->last_hw_ipcso = 0; - txr->last_hw_tucss = 0; - txr->last_hw_tucso = 0; + em_release_hw_control(adapter); - bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - EM_TX_UNLOCK(txr); + if (adapter->mta != NULL) { + free(adapter->mta, M_DEVBUF); + } } /********************************************************************* * - * Initialize all transmit rings. + * Enable transmit unit. * **********************************************************************/ static void -em_setup_transmit_structures(struct adapter *adapter) +em_initialize_transmit_unit(if_ctx_t ctx) { - struct tx_ring *txr = adapter->tx_rings; + struct adapter *adapter = iflib_get_softc(ctx); + if_softc_ctx_t scctx = adapter->shared; + struct em_tx_queue *que; + struct tx_ring *txr; + struct e1000_hw *hw = &adapter->hw; + u32 tctl, txdctl = 0, tarc, tipg = 0; - for (int i = 0; i < adapter->num_queues; i++, txr++) - em_setup_transmit_ring(txr); + INIT_DEBUGOUT("em_initialize_transmit_unit: begin"); - return; -} + for (int i = 0; i < adapter->tx_num_queues; i++, txr++) { + u64 bus_addr; + caddr_t offp, endp; -/********************************************************************* - * - * Enable transmit unit. - * - **********************************************************************/ -static void -em_initialize_transmit_unit(struct adapter *adapter) -{ - struct tx_ring *txr = adapter->tx_rings; - struct e1000_hw *hw = &adapter->hw; - u32 tctl, txdctl = 0, tarc, tipg = 0; + que = &adapter->tx_queues[i]; + txr = &que->txr; + bus_addr = txr->tx_paddr; - INIT_DEBUGOUT("em_initialize_transmit_unit: begin"); + /* Clear checksum offload context. */ + offp = (caddr_t)&txr->csum_flags; + endp = (caddr_t)(txr + 1); + bzero(offp, endp - offp); - for (int i = 0; i < adapter->num_queues; i++, txr++) { - u64 bus_addr = txr->txdma.dma_paddr; /* Base and Len of TX Ring */ E1000_WRITE_REG(hw, E1000_TDLEN(i), - adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); + scctx->isc_ntxd[0] * sizeof(struct e1000_tx_desc)); E1000_WRITE_REG(hw, E1000_TDBAH(i), - (u32)(bus_addr >> 32)); + (u32)(bus_addr >> 32)); E1000_WRITE_REG(hw, E1000_TDBAL(i), - (u32)bus_addr); + (u32)bus_addr); /* Init the HEAD/TAIL indices */ E1000_WRITE_REG(hw, E1000_TDT(i), 0); E1000_WRITE_REG(hw, E1000_TDH(i), 0); @@ -3683,16 +2818,15 @@ em_initialize_transmit_unit(struct adapter *adapter) E1000_READ_REG(&adapter->hw, E1000_TDBAL(i)), E1000_READ_REG(&adapter->hw, E1000_TDLEN(i))); - txr->busy = EM_TX_IDLE; txdctl = 0; /* clear txdctl */ - txdctl |= 0x1f; /* PTHRESH */ - txdctl |= 1 << 8; /* HTHRESH */ - txdctl |= 1 << 16;/* WTHRESH */ + txdctl |= 0x1f; /* PTHRESH */ + txdctl |= 1 << 8; /* HTHRESH */ + txdctl |= 1 << 16;/* WTHRESH */ txdctl |= 1 << 22; /* Reserved bit 22 must always be 1 */ txdctl |= E1000_TXDCTL_GRAN; - txdctl |= 1 << 25; /* LWTHRESH */ + txdctl |= 1 << 25; /* LWTHRESH */ - E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl); + E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl); } /* Set the default values for the Tx Inter Packet Gap timer */ @@ -3702,6 +2836,11 @@ em_initialize_transmit_unit(struct adapter *adapter) tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; break; + case e1000_82542: + tipg = DEFAULT_82542_TIPG_IPGT; + tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; + tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; + break; default: if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || (adapter->hw.phy.media_type == @@ -3736,7 +2875,7 @@ em_initialize_transmit_unit(struct adapter *adapter) } else if (adapter->hw.mac.type == e1000_82574) { tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); tarc |= TARC_ERRATA_BIT; - if ( adapter->num_queues > 1) { + if ( adapter->tx_num_queues > 1) { tarc |= (TARC_COMPENSATION_MODE | TARC_MQ_FIX); E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc); @@ -3744,7 +2883,6 @@ em_initialize_transmit_unit(struct adapter *adapter) E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); } - adapter->txd_cmd = E1000_TXD_CMD_IFCS; if (adapter->tx_int_delay.value > 0) adapter->txd_cmd |= E1000_TXD_CMD_IDE; @@ -3771,780 +2909,6 @@ em_initialize_transmit_unit(struct adapter *adapter) } } - -/********************************************************************* - * - * Free all transmit rings. - * - **********************************************************************/ -static void -em_free_transmit_structures(struct adapter *adapter) -{ - struct tx_ring *txr = adapter->tx_rings; - - for (int i = 0; i < adapter->num_queues; i++, txr++) { - EM_TX_LOCK(txr); - em_free_transmit_buffers(txr); - em_dma_free(adapter, &txr->txdma); - EM_TX_UNLOCK(txr); - EM_TX_LOCK_DESTROY(txr); - } - - free(adapter->tx_rings, M_DEVBUF); -} - -/********************************************************************* - * - * Free transmit ring related data structures. - * - **********************************************************************/ -static void -em_free_transmit_buffers(struct tx_ring *txr) -{ - struct adapter *adapter = txr->adapter; - struct em_txbuffer *txbuf; - - INIT_DEBUGOUT("free_transmit_ring: begin"); - - if (txr->tx_buffers == NULL) - return; - - for (int i = 0; i < adapter->num_tx_desc; i++) { - txbuf = &txr->tx_buffers[i]; - if (txbuf->m_head != NULL) { - bus_dmamap_sync(txr->txtag, txbuf->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(txr->txtag, - txbuf->map); - m_freem(txbuf->m_head); - txbuf->m_head = NULL; - if (txbuf->map != NULL) { - bus_dmamap_destroy(txr->txtag, - txbuf->map); - txbuf->map = NULL; - } - } else if (txbuf->map != NULL) { - bus_dmamap_unload(txr->txtag, - txbuf->map); - bus_dmamap_destroy(txr->txtag, - txbuf->map); - txbuf->map = NULL; - } - } -#if __FreeBSD_version >= 800000 - if (txr->br != NULL) - buf_ring_free(txr->br, M_DEVBUF); -#endif - if (txr->tx_buffers != NULL) { - free(txr->tx_buffers, M_DEVBUF); - txr->tx_buffers = NULL; - } - if (txr->txtag != NULL) { - bus_dma_tag_destroy(txr->txtag); - txr->txtag = NULL; - } - return; -} - - -/********************************************************************* - * The offload context is protocol specific (TCP/UDP) and thus - * only needs to be set when the protocol changes. The occasion - * of a context change can be a performance detriment, and - * might be better just disabled. The reason arises in the way - * in which the controller supports pipelined requests from the - * Tx data DMA. Up to four requests can be pipelined, and they may - * belong to the same packet or to multiple packets. However all - * requests for one packet are issued before a request is issued - * for a subsequent packet and if a request for the next packet - * requires a context change, that request will be stalled - * until the previous request completes. This means setting up - * a new context effectively disables pipelined Tx data DMA which - * in turn greatly slow down performance to send small sized - * frames. - **********************************************************************/ -static void -em_transmit_checksum_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off, - struct ip *ip, u32 *txd_upper, u32 *txd_lower) -{ - struct adapter *adapter = txr->adapter; - struct e1000_context_desc *TXD = NULL; - struct em_txbuffer *tx_buffer; - int cur, hdr_len; - u32 cmd = 0; - u16 offload = 0; - u8 ipcso, ipcss, tucso, tucss; - - ipcss = ipcso = tucss = tucso = 0; - hdr_len = ip_off + (ip->ip_hl << 2); - cur = txr->next_avail_desc; - - /* Setup of IP header checksum. */ - if (mp->m_pkthdr.csum_flags & CSUM_IP) { - *txd_upper |= E1000_TXD_POPTS_IXSM << 8; - offload |= CSUM_IP; - ipcss = ip_off; - ipcso = ip_off + offsetof(struct ip, ip_sum); - /* - * Start offset for header checksum calculation. - * End offset for header checksum calculation. - * Offset of place to put the checksum. - */ - TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; - TXD->lower_setup.ip_fields.ipcss = ipcss; - TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len); - TXD->lower_setup.ip_fields.ipcso = ipcso; - cmd |= E1000_TXD_CMD_IP; - } - - if (mp->m_pkthdr.csum_flags & CSUM_TCP) { - *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; - *txd_upper |= E1000_TXD_POPTS_TXSM << 8; - offload |= CSUM_TCP; - tucss = hdr_len; - tucso = hdr_len + offsetof(struct tcphdr, th_sum); - /* - * The 82574L can only remember the *last* context used - * regardless of queue that it was use for. We cannot reuse - * contexts on this hardware platform and must generate a new - * context every time. 82574L hardware spec, section 7.2.6, - * second note. - */ - if (adapter->num_queues < 2) { - /* - * Setting up new checksum offload context for every - * frames takes a lot of processing time for hardware. - * This also reduces performance a lot for small sized - * frames so avoid it if driver can use previously - * configured checksum offload context. - */ - if (txr->last_hw_offload == offload) { - if (offload & CSUM_IP) { - if (txr->last_hw_ipcss == ipcss && - txr->last_hw_ipcso == ipcso && - txr->last_hw_tucss == tucss && - txr->last_hw_tucso == tucso) - return; - } else { - if (txr->last_hw_tucss == tucss && - txr->last_hw_tucso == tucso) - return; - } - } - txr->last_hw_offload = offload; - txr->last_hw_tucss = tucss; - txr->last_hw_tucso = tucso; - } - /* - * Start offset for payload checksum calculation. - * End offset for payload checksum calculation. - * Offset of place to put the checksum. - */ - TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; - TXD->upper_setup.tcp_fields.tucss = hdr_len; - TXD->upper_setup.tcp_fields.tucse = htole16(0); - TXD->upper_setup.tcp_fields.tucso = tucso; - cmd |= E1000_TXD_CMD_TCP; - } else if (mp->m_pkthdr.csum_flags & CSUM_UDP) { - *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; - *txd_upper |= E1000_TXD_POPTS_TXSM << 8; - tucss = hdr_len; - tucso = hdr_len + offsetof(struct udphdr, uh_sum); - /* - * The 82574L can only remember the *last* context used - * regardless of queue that it was use for. We cannot reuse - * contexts on this hardware platform and must generate a new - * context every time. 82574L hardware spec, section 7.2.6, - * second note. - */ - if (adapter->num_queues < 2) { - /* - * Setting up new checksum offload context for every - * frames takes a lot of processing time for hardware. - * This also reduces performance a lot for small sized - * frames so avoid it if driver can use previously - * configured checksum offload context. - */ - if (txr->last_hw_offload == offload) { - if (offload & CSUM_IP) { - if (txr->last_hw_ipcss == ipcss && - txr->last_hw_ipcso == ipcso && - txr->last_hw_tucss == tucss && - txr->last_hw_tucso == tucso) - return; - } else { - if (txr->last_hw_tucss == tucss && - txr->last_hw_tucso == tucso) - return; - } - } - txr->last_hw_offload = offload; - txr->last_hw_tucss = tucss; - txr->last_hw_tucso = tucso; - } - /* - * Start offset for header checksum calculation. - * End offset for header checksum calculation. - * Offset of place to put the checksum. - */ - TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; - TXD->upper_setup.tcp_fields.tucss = tucss; - TXD->upper_setup.tcp_fields.tucse = htole16(0); - TXD->upper_setup.tcp_fields.tucso = tucso; - } - - if (offload & CSUM_IP) { - txr->last_hw_ipcss = ipcss; - txr->last_hw_ipcso = ipcso; - } - - TXD->tcp_seg_setup.data = htole32(0); - TXD->cmd_and_length = - htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd); - tx_buffer = &txr->tx_buffers[cur]; - tx_buffer->m_head = NULL; - tx_buffer->next_eop = -1; - - if (++cur == adapter->num_tx_desc) - cur = 0; - - txr->tx_avail--; - txr->next_avail_desc = cur; -} - - -/********************************************************************** - * - * Setup work for hardware segmentation offload (TSO) - * - **********************************************************************/ -static void -em_tso_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off, - struct ip *ip, struct tcphdr *tp, u32 *txd_upper, u32 *txd_lower) -{ - struct adapter *adapter = txr->adapter; - struct e1000_context_desc *TXD; - struct em_txbuffer *tx_buffer; - int cur, hdr_len; - - /* - * In theory we can use the same TSO context if and only if - * frame is the same type(IP/TCP) and the same MSS. However - * checking whether a frame has the same IP/TCP structure is - * hard thing so just ignore that and always restablish a - * new TSO context. - */ - hdr_len = ip_off + (ip->ip_hl << 2) + (tp->th_off << 2); - *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ - E1000_TXD_DTYP_D | /* Data descr type */ - E1000_TXD_CMD_TSE); /* Do TSE on this packet */ - - /* IP and/or TCP header checksum calculation and insertion. */ - *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8; - - cur = txr->next_avail_desc; - tx_buffer = &txr->tx_buffers[cur]; - TXD = (struct e1000_context_desc *) &txr->tx_base[cur]; - - /* - * Start offset for header checksum calculation. - * End offset for header checksum calculation. - * Offset of place put the checksum. - */ - TXD->lower_setup.ip_fields.ipcss = ip_off; - TXD->lower_setup.ip_fields.ipcse = - htole16(ip_off + (ip->ip_hl << 2) - 1); - TXD->lower_setup.ip_fields.ipcso = ip_off + offsetof(struct ip, ip_sum); - /* - * Start offset for payload checksum calculation. - * End offset for payload checksum calculation. - * Offset of place to put the checksum. - */ - TXD->upper_setup.tcp_fields.tucss = ip_off + (ip->ip_hl << 2); - TXD->upper_setup.tcp_fields.tucse = 0; - TXD->upper_setup.tcp_fields.tucso = - ip_off + (ip->ip_hl << 2) + offsetof(struct tcphdr, th_sum); - /* - * Payload size per packet w/o any headers. - * Length of all headers up to payload. - */ - TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz); - TXD->tcp_seg_setup.fields.hdr_len = hdr_len; - - TXD->cmd_and_length = htole32(adapter->txd_cmd | - E1000_TXD_CMD_DEXT | /* Extended descr */ - E1000_TXD_CMD_TSE | /* TSE context */ - E1000_TXD_CMD_IP | /* Do IP csum */ - E1000_TXD_CMD_TCP | /* Do TCP checksum */ - (mp->m_pkthdr.len - (hdr_len))); /* Total len */ - - tx_buffer->m_head = NULL; - tx_buffer->next_eop = -1; - - if (++cur == adapter->num_tx_desc) - cur = 0; - - txr->tx_avail--; - txr->next_avail_desc = cur; - txr->tx_tso = TRUE; -} - - -/********************************************************************** - * - * Examine each tx_buffer in the used queue. If the hardware is done - * processing the packet then free associated resources. The - * tx_buffer is put back on the free queue. - * - **********************************************************************/ -static void -em_txeof(struct tx_ring *txr) -{ - struct adapter *adapter = txr->adapter; - int first, last, done, processed; - struct em_txbuffer *tx_buffer; - struct e1000_tx_desc *tx_desc, *eop_desc; - if_t ifp = adapter->ifp; - - EM_TX_LOCK_ASSERT(txr); -#ifdef DEV_NETMAP - if (netmap_tx_irq(ifp, txr->me)) - return; -#endif /* DEV_NETMAP */ - - /* No work, make sure hang detection is disabled */ - if (txr->tx_avail == adapter->num_tx_desc) { - txr->busy = EM_TX_IDLE; - return; - } - - processed = 0; - first = txr->next_to_clean; - tx_desc = &txr->tx_base[first]; - tx_buffer = &txr->tx_buffers[first]; - last = tx_buffer->next_eop; - eop_desc = &txr->tx_base[last]; - - /* - * What this does is get the index of the - * first descriptor AFTER the EOP of the - * first packet, that way we can do the - * simple comparison on the inner while loop. - */ - if (++last == adapter->num_tx_desc) - last = 0; - done = last; - - bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, - BUS_DMASYNC_POSTREAD); - - while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) { - /* We clean the range of the packet */ - while (first != done) { - tx_desc->upper.data = 0; - tx_desc->lower.data = 0; - tx_desc->buffer_addr = 0; - ++txr->tx_avail; - ++processed; - - if (tx_buffer->m_head) { - bus_dmamap_sync(txr->txtag, - tx_buffer->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(txr->txtag, - tx_buffer->map); - m_freem(tx_buffer->m_head); - tx_buffer->m_head = NULL; - } - tx_buffer->next_eop = -1; - - if (++first == adapter->num_tx_desc) - first = 0; - - tx_buffer = &txr->tx_buffers[first]; - tx_desc = &txr->tx_base[first]; - } - if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); - /* See if we can continue to the next packet */ - last = tx_buffer->next_eop; - if (last != -1) { - eop_desc = &txr->tx_base[last]; - /* Get new done point */ - if (++last == adapter->num_tx_desc) last = 0; - done = last; - } else - break; - } - bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - - txr->next_to_clean = first; - - /* - ** Hang detection: we know there's work outstanding - ** or the entry return would have been taken, so no - ** descriptor processed here indicates a potential hang. - ** The local timer will examine this and do a reset if needed. - */ - if (processed == 0) { - if (txr->busy != EM_TX_HUNG) - ++txr->busy; - } else /* At least one descriptor was cleaned */ - txr->busy = EM_TX_BUSY; /* note this clears HUNG */ - - /* - * If we have a minimum free, clear IFF_DRV_OACTIVE - * to tell the stack that it is OK to send packets. - * Notice that all writes of OACTIVE happen under the - * TX lock which, with a single queue, guarantees - * sanity. - */ - if (txr->tx_avail >= EM_MAX_SCATTER) { - if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); - } - - /* Disable hang detection if all clean */ - if (txr->tx_avail == adapter->num_tx_desc) - txr->busy = EM_TX_IDLE; -} - -/********************************************************************* - * - * Refresh RX descriptor mbufs from system mbuf buffer pool. - * - **********************************************************************/ -static void -em_refresh_mbufs(struct rx_ring *rxr, int limit) -{ - struct adapter *adapter = rxr->adapter; - struct mbuf *m; - bus_dma_segment_t segs; - struct em_rxbuffer *rxbuf; - int i, j, error, nsegs; - bool cleaned = FALSE; - - i = j = rxr->next_to_refresh; - /* - ** Get one descriptor beyond - ** our work mark to control - ** the loop. - */ - if (++j == adapter->num_rx_desc) - j = 0; - - while (j != limit) { - rxbuf = &rxr->rx_buffers[i]; - if (rxbuf->m_head == NULL) { - m = m_getjcl(M_NOWAIT, MT_DATA, - M_PKTHDR, adapter->rx_mbuf_sz); - /* - ** If we have a temporary resource shortage - ** that causes a failure, just abort refresh - ** for now, we will return to this point when - ** reinvoked from em_rxeof. - */ - if (m == NULL) - goto update; - } else - m = rxbuf->m_head; - - m->m_len = m->m_pkthdr.len = adapter->rx_mbuf_sz; - m->m_flags |= M_PKTHDR; - m->m_data = m->m_ext.ext_buf; - - /* Use bus_dma machinery to setup the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->rxtag, rxbuf->map, - m, &segs, &nsegs, BUS_DMA_NOWAIT); - if (error != 0) { - printf("Refresh mbufs: hdr dmamap load" - " failure - %d\n", error); - m_free(m); - rxbuf->m_head = NULL; - goto update; - } - rxbuf->m_head = m; - rxbuf->paddr = segs.ds_addr; - bus_dmamap_sync(rxr->rxtag, - rxbuf->map, BUS_DMASYNC_PREREAD); - em_setup_rxdesc(&rxr->rx_base[i], rxbuf); - cleaned = TRUE; - - i = j; /* Next is precalulated for us */ - rxr->next_to_refresh = i; - /* Calculate next controlling index */ - if (++j == adapter->num_rx_desc) - j = 0; - } -update: - /* - ** Update the tail pointer only if, - ** and as far as we have refreshed. - */ - if (cleaned) - E1000_WRITE_REG(&adapter->hw, - E1000_RDT(rxr->me), rxr->next_to_refresh); - - return; -} - - -/********************************************************************* - * - * Allocate memory for rx_buffer structures. Since we use one - * rx_buffer per received packet, the maximum number of rx_buffer's - * that we'll need is equal to the number of receive descriptors - * that we've allocated. - * - **********************************************************************/ -static int -em_allocate_receive_buffers(struct rx_ring *rxr) -{ - struct adapter *adapter = rxr->adapter; - device_t dev = adapter->dev; - struct em_rxbuffer *rxbuf; - int error; - - rxr->rx_buffers = malloc(sizeof(struct em_rxbuffer) * - adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); - if (rxr->rx_buffers == NULL) { - device_printf(dev, "Unable to allocate rx_buffer memory\n"); - return (ENOMEM); - } - - error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - MJUM9BYTES, /* maxsize */ - 1, /* nsegments */ - MJUM9BYTES, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockarg */ - &rxr->rxtag); - if (error) { - device_printf(dev, "%s: bus_dma_tag_create failed %d\n", - __func__, error); - goto fail; - } - - rxbuf = rxr->rx_buffers; - for (int i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { - rxbuf = &rxr->rx_buffers[i]; - error = bus_dmamap_create(rxr->rxtag, 0, &rxbuf->map); - if (error) { - device_printf(dev, "%s: bus_dmamap_create failed: %d\n", - __func__, error); - goto fail; - } - } - - return (0); - -fail: - em_free_receive_structures(adapter); - return (error); -} - - -/********************************************************************* - * - * Initialize a receive ring and its buffers. - * - **********************************************************************/ -static int -em_setup_receive_ring(struct rx_ring *rxr) -{ - struct adapter *adapter = rxr->adapter; - struct em_rxbuffer *rxbuf; - bus_dma_segment_t seg[1]; - int rsize, nsegs, error = 0; -#ifdef DEV_NETMAP - struct netmap_slot *slot; - struct netmap_adapter *na = netmap_getna(adapter->ifp); -#endif - - - /* Clear the ring contents */ - EM_RX_LOCK(rxr); - rsize = roundup2(adapter->num_rx_desc * - sizeof(union e1000_rx_desc_extended), EM_DBA_ALIGN); - bzero((void *)rxr->rx_base, rsize); -#ifdef DEV_NETMAP - slot = netmap_reset(na, NR_RX, rxr->me, 0); -#endif - - /* - ** Free current RX buffer structs and their mbufs - */ - for (int i = 0; i < adapter->num_rx_desc; i++) { - rxbuf = &rxr->rx_buffers[i]; - if (rxbuf->m_head != NULL) { - bus_dmamap_sync(rxr->rxtag, rxbuf->map, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->rxtag, rxbuf->map); - m_freem(rxbuf->m_head); - rxbuf->m_head = NULL; /* mark as freed */ - } - } - - /* Now replenish the mbufs */ - for (int j = 0; j != adapter->num_rx_desc; ++j) { - rxbuf = &rxr->rx_buffers[j]; -#ifdef DEV_NETMAP - if (slot) { - int si = netmap_idx_n2k(&na->rx_rings[rxr->me], j); - uint64_t paddr; - void *addr; - - addr = PNMB(na, slot + si, &paddr); - netmap_load_map(na, rxr->rxtag, rxbuf->map, addr); - rxbuf->paddr = paddr; - em_setup_rxdesc(&rxr->rx_base[j], rxbuf); - continue; - } -#endif /* DEV_NETMAP */ - rxbuf->m_head = m_getjcl(M_NOWAIT, MT_DATA, - M_PKTHDR, adapter->rx_mbuf_sz); - if (rxbuf->m_head == NULL) { - error = ENOBUFS; - goto fail; - } - rxbuf->m_head->m_len = adapter->rx_mbuf_sz; - rxbuf->m_head->m_flags &= ~M_HASFCS; /* we strip it */ - rxbuf->m_head->m_pkthdr.len = adapter->rx_mbuf_sz; - - /* Get the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->rxtag, - rxbuf->map, rxbuf->m_head, seg, - &nsegs, BUS_DMA_NOWAIT); - if (error != 0) { - m_freem(rxbuf->m_head); - rxbuf->m_head = NULL; - goto fail; - } - bus_dmamap_sync(rxr->rxtag, - rxbuf->map, BUS_DMASYNC_PREREAD); - - rxbuf->paddr = seg[0].ds_addr; - em_setup_rxdesc(&rxr->rx_base[j], rxbuf); - } - rxr->next_to_check = 0; - rxr->next_to_refresh = 0; - bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - -fail: - EM_RX_UNLOCK(rxr); - return (error); -} - -/********************************************************************* - * - * Initialize all receive rings. - * - **********************************************************************/ -static int -em_setup_receive_structures(struct adapter *adapter) -{ - struct rx_ring *rxr = adapter->rx_rings; - int q; - - for (q = 0; q < adapter->num_queues; q++, rxr++) - if (em_setup_receive_ring(rxr)) - goto fail; - - return (0); -fail: - /* - * Free RX buffers allocated so far, we will only handle - * the rings that completed, the failing case will have - * cleaned up for itself. 'q' failed, so its the terminus. - */ - for (int i = 0; i < q; ++i) { - rxr = &adapter->rx_rings[i]; - for (int n = 0; n < adapter->num_rx_desc; n++) { - struct em_rxbuffer *rxbuf; - rxbuf = &rxr->rx_buffers[n]; - if (rxbuf->m_head != NULL) { - bus_dmamap_sync(rxr->rxtag, rxbuf->map, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->rxtag, rxbuf->map); - m_freem(rxbuf->m_head); - rxbuf->m_head = NULL; - } - } - rxr->next_to_check = 0; - rxr->next_to_refresh = 0; - } - - return (ENOBUFS); -} - -/********************************************************************* - * - * Free all receive rings. - * - **********************************************************************/ -static void -em_free_receive_structures(struct adapter *adapter) -{ - struct rx_ring *rxr = adapter->rx_rings; - - for (int i = 0; i < adapter->num_queues; i++, rxr++) { - em_free_receive_buffers(rxr); - /* Free the ring memory as well */ - em_dma_free(adapter, &rxr->rxdma); - EM_RX_LOCK_DESTROY(rxr); - } - - free(adapter->rx_rings, M_DEVBUF); -} - - -/********************************************************************* - * - * Free receive ring data structures - * - **********************************************************************/ -static void -em_free_receive_buffers(struct rx_ring *rxr) -{ - struct adapter *adapter = rxr->adapter; - struct em_rxbuffer *rxbuf = NULL; - - INIT_DEBUGOUT("free_receive_buffers: begin"); - - if (rxr->rx_buffers != NULL) { - for (int i = 0; i < adapter->num_rx_desc; i++) { - rxbuf = &rxr->rx_buffers[i]; - if (rxbuf->map != NULL) { - bus_dmamap_sync(rxr->rxtag, rxbuf->map, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->rxtag, rxbuf->map); - bus_dmamap_destroy(rxr->rxtag, rxbuf->map); - } - if (rxbuf->m_head != NULL) { - m_freem(rxbuf->m_head); - rxbuf->m_head = NULL; - } - } - free(rxr->rx_buffers, M_DEVBUF); - rxr->rx_buffers = NULL; - rxr->next_to_check = 0; - rxr->next_to_refresh = 0; - } - - if (rxr->rxtag != NULL) { - bus_dma_tag_destroy(rxr->rxtag); - rxr->rxtag = NULL; - } - - return; -} - - /********************************************************************* * * Enable receive unit. @@ -4552,12 +2916,15 @@ em_free_receive_buffers(struct rx_ring *rxr) **********************************************************************/ static void -em_initialize_receive_unit(struct adapter *adapter) +em_initialize_receive_unit(if_ctx_t ctx) { - struct rx_ring *rxr = adapter->rx_rings; - if_t ifp = adapter->ifp; + struct adapter *adapter = iflib_get_softc(ctx); + if_softc_ctx_t scctx = adapter->shared; + struct ifnet *ifp = iflib_get_ifp(ctx); struct e1000_hw *hw = &adapter->hw; - u32 rctl, rxcsum, rfctl; + struct em_rx_queue *que; + int i; + u32 rctl, rxcsum, rfctl; INIT_DEBUGOUT("em_initialize_receive_units: begin"); @@ -4585,28 +2952,30 @@ em_initialize_receive_unit(struct adapter *adapter) else rctl &= ~E1000_RCTL_LPE; - /* Strip the CRC */ - if (!em_disable_crc_stripping) + /* Strip the CRC */ + if (!em_disable_crc_stripping) rctl |= E1000_RCTL_SECRC; - E1000_WRITE_REG(&adapter->hw, E1000_RADV, - adapter->rx_abs_int_delay.value); + if (adapter->hw.mac.type >= e1000_82540) { + E1000_WRITE_REG(&adapter->hw, E1000_RADV, + adapter->rx_abs_int_delay.value); + /* + * Set the interrupt throttling rate. Value is calculated + * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) + */ + E1000_WRITE_REG(hw, E1000_ITR, DEFAULT_ITR); + } E1000_WRITE_REG(&adapter->hw, E1000_RDTR, adapter->rx_int_delay.value); - /* - * Set the interrupt throttling rate. Value is calculated - * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) - */ - E1000_WRITE_REG(hw, E1000_ITR, DEFAULT_ITR); /* Use extended rx descriptor formats */ rfctl = E1000_READ_REG(hw, E1000_RFCTL); rfctl |= E1000_RFCTL_EXTEN; /* - ** When using MSIX interrupts we need to throttle - ** using the EITR register (82574 only) - */ + * When using MSIX interrupts we need to throttle + * using the EITR register (82574 only) + */ if (hw->mac.type == e1000_82574) { for (int i = 0; i < 4; i++) E1000_WRITE_REG(hw, E1000_EITR_82574(i), @@ -4617,92 +2986,62 @@ em_initialize_receive_unit(struct adapter *adapter) E1000_WRITE_REG(hw, E1000_RFCTL, rfctl); rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); - if (if_getcapenable(ifp) & IFCAP_RXCSUM) { -#ifdef EM_MULTIQUEUE - rxcsum |= E1000_RXCSUM_TUOFL | - E1000_RXCSUM_IPOFL | - E1000_RXCSUM_PCSD; -#else - rxcsum |= E1000_RXCSUM_TUOFL; -#endif + if (if_getcapenable(ifp) & IFCAP_RXCSUM && + adapter->hw.mac.type >= e1000_82543) { + if (adapter->tx_num_queues > 1) { + if (adapter->hw.mac.type >= igb_mac_min) { + rxcsum |= E1000_RXCSUM_PCSD; + if (hw->mac.type != e1000_82575) + rxcsum |= E1000_RXCSUM_CRCOFL; + } else + rxcsum |= E1000_RXCSUM_TUOFL | + E1000_RXCSUM_IPOFL | + E1000_RXCSUM_PCSD; + } else { + if (adapter->hw.mac.type >= igb_mac_min) + rxcsum |= E1000_RXCSUM_IPPCSE; + else + rxcsum |= E1000_RXCSUM_TUOFL | E1000_RXCSUM_IPOFL; + if (adapter->hw.mac.type > e1000_82575) + rxcsum |= E1000_RXCSUM_CRCOFL; + } } else rxcsum &= ~E1000_RXCSUM_TUOFL; E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); -#ifdef EM_MULTIQUEUE -#define RSSKEYLEN 10 - if (adapter->num_queues > 1) { - uint8_t rss_key[4 * RSSKEYLEN]; - uint32_t reta = 0; - int i; - - /* - * Configure RSS key - */ - arc4rand(rss_key, sizeof(rss_key), 0); - for (i = 0; i < RSSKEYLEN; ++i) { - uint32_t rssrk = 0; - - rssrk = EM_RSSRK_VAL(rss_key, i); - E1000_WRITE_REG(hw,E1000_RSSRK(i), rssrk); - } - - /* - * Configure RSS redirect table in following fashion: - * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)] - */ - for (i = 0; i < sizeof(reta); ++i) { - uint32_t q; - - q = (i % adapter->num_queues) << 7; - reta |= q << (8 * i); - } - - for (i = 0; i < 32; ++i) { - E1000_WRITE_REG(hw, E1000_RETA(i), reta); - } - - E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_RSS_ENABLE_2Q | - E1000_MRQC_RSS_FIELD_IPV4_TCP | - E1000_MRQC_RSS_FIELD_IPV4 | - E1000_MRQC_RSS_FIELD_IPV6_TCP_EX | - E1000_MRQC_RSS_FIELD_IPV6_EX | - E1000_MRQC_RSS_FIELD_IPV6); + if (adapter->rx_num_queues > 1) { + if (adapter->hw.mac.type >= igb_mac_min) + igb_initialize_rss_mapping(adapter); + else + em_initialize_rss_mapping(adapter); } -#endif + /* - ** XXX TEMPORARY WORKAROUND: on some systems with 82573 - ** long latencies are observed, like Lenovo X60. This - ** change eliminates the problem, but since having positive - ** values in RDTR is a known source of problems on other - ** platforms another solution is being sought. - */ + * XXX TEMPORARY WORKAROUND: on some systems with 82573 + * long latencies are observed, like Lenovo X60. This + * change eliminates the problem, but since having positive + * values in RDTR is a known source of problems on other + * platforms another solution is being sought. + */ if (hw->mac.type == e1000_82573) E1000_WRITE_REG(hw, E1000_RDTR, 0x20); - for (int i = 0; i < adapter->num_queues; i++, rxr++) { + for (i = 0, que = adapter->rx_queues; i < adapter->rx_num_queues; i++, que++) { + struct rx_ring *rxr = &que->rxr; /* Setup the Base and Length of the Rx Descriptor Ring */ - u64 bus_addr = rxr->rxdma.dma_paddr; - u32 rdt = adapter->num_rx_desc - 1; /* default */ + u64 bus_addr = rxr->rx_paddr; +#if 0 + u32 rdt = adapter->rx_num_queues -1; /* default */ +#endif E1000_WRITE_REG(hw, E1000_RDLEN(i), - adapter->num_rx_desc * sizeof(union e1000_rx_desc_extended)); + scctx->isc_nrxd[0] * sizeof(union e1000_rx_desc_extended)); E1000_WRITE_REG(hw, E1000_RDBAH(i), (u32)(bus_addr >> 32)); E1000_WRITE_REG(hw, E1000_RDBAL(i), (u32)bus_addr); /* Setup the Head and Tail Descriptor Pointers */ E1000_WRITE_REG(hw, E1000_RDH(i), 0); -#ifdef DEV_NETMAP - /* - * an init() while a netmap client is active must - * preserve the rx buffers passed to userspace. - */ - if (if_getcapenable(ifp) & IFCAP_NETMAP) { - struct netmap_adapter *na = netmap_getna(adapter->ifp); - rdt -= nm_kr_rxspace(&na->rx_rings[i]); - } -#endif /* DEV_NETMAP */ - E1000_WRITE_REG(hw, E1000_RDT(i), rdt); + E1000_WRITE_REG(hw, E1000_RDT(i), 0); } /* @@ -4712,6 +3051,7 @@ em_initialize_receive_unit(struct adapter *adapter) * Only write to RXDCTL(1) if there is a need for different * settings. */ + if (((adapter->hw.mac.type == e1000_ich9lan) || (adapter->hw.mac.type == e1000_pch2lan) || (adapter->hw.mac.type == e1000_ich10lan)) && @@ -4719,384 +3059,151 @@ em_initialize_receive_unit(struct adapter *adapter) u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0)); E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl | 3); } else if (adapter->hw.mac.type == e1000_82574) { - for (int i = 0; i < adapter->num_queues; i++) { + for (int i = 0; i < adapter->rx_num_queues; i++) { u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); - rxdctl |= 0x20; /* PTHRESH */ rxdctl |= 4 << 8; /* HTHRESH */ rxdctl |= 4 << 16;/* WTHRESH */ rxdctl |= 1 << 24; /* Switch to granularity */ E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); } - } - - if (adapter->hw.mac.type >= e1000_pch2lan) { - if (if_getmtu(ifp) > ETHERMTU) - e1000_lv_jumbo_workaround_ich8lan(hw, TRUE); - else - e1000_lv_jumbo_workaround_ich8lan(hw, FALSE); - } - - /* Make sure VLAN Filters are off */ - rctl &= ~E1000_RCTL_VFE; - - if (adapter->rx_mbuf_sz == MCLBYTES) - rctl |= E1000_RCTL_SZ_2048; - else if (adapter->rx_mbuf_sz == MJUMPAGESIZE) - rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX; - else if (adapter->rx_mbuf_sz > MJUMPAGESIZE) - rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX; - - /* ensure we clear use DTYPE of 00 here */ - rctl &= ~0x00000C00; - /* Write out the settings */ - E1000_WRITE_REG(hw, E1000_RCTL, rctl); - - return; -} - - -/********************************************************************* - * - * This routine executes in interrupt context. It replenishes - * the mbufs in the descriptor and sends data which has been - * dma'ed into host memory to upper layer. - * - * We loop at most count times if count is > 0, or until done if - * count < 0. - * - * For polling we also now return the number of cleaned packets - *********************************************************************/ -static bool -em_rxeof(struct rx_ring *rxr, int count, int *done) -{ - struct adapter *adapter = rxr->adapter; - if_t ifp = adapter->ifp; - struct mbuf *mp, *sendmp; - u32 status = 0; - u16 len; - int i, processed, rxdone = 0; - bool eop; - union e1000_rx_desc_extended *cur; - - EM_RX_LOCK(rxr); - - /* Sync the ring */ - bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - - -#ifdef DEV_NETMAP - if (netmap_rx_irq(ifp, rxr->me, &processed)) { - EM_RX_UNLOCK(rxr); - return (FALSE); - } -#endif /* DEV_NETMAP */ - - for (i = rxr->next_to_check, processed = 0; count != 0;) { - if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) - break; - - cur = &rxr->rx_base[i]; - status = le32toh(cur->wb.upper.status_error); - mp = sendmp = NULL; - - if ((status & E1000_RXD_STAT_DD) == 0) - break; - - len = le16toh(cur->wb.upper.length); - eop = (status & E1000_RXD_STAT_EOP) != 0; - - if ((status & E1000_RXDEXT_ERR_FRAME_ERR_MASK) || - (rxr->discard == TRUE)) { - adapter->dropped_pkts++; - ++rxr->rx_discarded; - if (!eop) /* Catch subsequent segs */ - rxr->discard = TRUE; - else - rxr->discard = FALSE; - em_rx_discard(rxr, i); - goto next_desc; - } - bus_dmamap_unload(rxr->rxtag, rxr->rx_buffers[i].map); - - /* Assign correct length to the current fragment */ - mp = rxr->rx_buffers[i].m_head; - mp->m_len = len; + } else if (adapter->hw.mac.type >= igb_mac_min) { + u32 psize, srrctl = 0; - /* Trigger for refresh */ - rxr->rx_buffers[i].m_head = NULL; - - /* First segment? */ - if (rxr->fmp == NULL) { - mp->m_pkthdr.len = len; - rxr->fmp = rxr->lmp = mp; - } else { - /* Chain mbuf's together */ - mp->m_flags &= ~M_PKTHDR; - rxr->lmp->m_next = mp; - rxr->lmp = mp; - rxr->fmp->m_pkthdr.len += len; - } - - if (eop) { - --count; - sendmp = rxr->fmp; - if_setrcvif(sendmp, ifp); - if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); - em_receive_checksum(status, sendmp); -#ifndef __NO_STRICT_ALIGNMENT - if (adapter->hw.mac.max_frame_size > - (MCLBYTES - ETHER_ALIGN) && - em_fixup_rx(rxr) != 0) - goto skip; -#endif - if (status & E1000_RXD_STAT_VP) { - if_setvtag(sendmp, - le16toh(cur->wb.upper.vlan)); - sendmp->m_flags |= M_VLANTAG; + if (if_getmtu(ifp) > ETHERMTU) { + /* Set maximum packet len */ + if (adapter->rx_mbuf_sz <= 4096) { + srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT; + rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX; + } else if (adapter->rx_mbuf_sz > 4096) { + srrctl |= 8192 >> E1000_SRRCTL_BSIZEPKT_SHIFT; + rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX; } -#ifndef __NO_STRICT_ALIGNMENT -skip: -#endif - rxr->fmp = rxr->lmp = NULL; + psize = scctx->isc_max_frame_size; + /* are we on a vlan? */ + if (ifp->if_vlantrunk != NULL) + psize += VLAN_TAG_SIZE; + E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize); + } else { + srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT; + rctl |= E1000_RCTL_SZ_2048; } -next_desc: - /* Sync the ring */ - bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - - /* Zero out the receive descriptors status. */ - cur->wb.upper.status_error &= htole32(~0xFF); - ++rxdone; /* cumulative for POLL */ - ++processed; - - /* Advance our pointers to the next descriptor. */ - if (++i == adapter->num_rx_desc) - i = 0; - - /* Send to the stack */ - if (sendmp != NULL) { - rxr->next_to_check = i; - EM_RX_UNLOCK(rxr); - if_input(ifp, sendmp); - EM_RX_LOCK(rxr); - i = rxr->next_to_check; + + /* + * If TX flow control is disabled and there's >1 queue defined, + * enable DROP. + * + * This drops frames rather than hanging the RX MAC for all queues. + */ + if ((adapter->rx_num_queues > 1) && + (adapter->fc == e1000_fc_none || + adapter->fc == e1000_fc_rx_pause)) { + srrctl |= E1000_SRRCTL_DROP_EN; } + /* Setup the Base and Length of the Rx Descriptor Rings */ + for (i = 0, que = adapter->rx_queues; i < adapter->rx_num_queues; i++, que++) { + struct rx_ring *rxr = &que->rxr; + u64 bus_addr = rxr->rx_paddr; + u32 rxdctl; + +#ifdef notyet + /* Configure for header split? -- ignore for now */ + rxr->hdr_split = igb_header_split; +#else + srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF; +#endif - /* Only refresh mbufs every 8 descriptors */ - if (processed == 8) { - em_refresh_mbufs(rxr, i); - processed = 0; - } + E1000_WRITE_REG(hw, E1000_RDLEN(i), + scctx->isc_nrxd[0] * sizeof(struct e1000_rx_desc)); + E1000_WRITE_REG(hw, E1000_RDBAH(i), + (uint32_t)(bus_addr >> 32)); + E1000_WRITE_REG(hw, E1000_RDBAL(i), + (uint32_t)bus_addr); + E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl); + /* Enable this Queue */ + rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); + rxdctl |= E1000_RXDCTL_QUEUE_ENABLE; + rxdctl &= 0xFFF00000; + rxdctl |= IGB_RX_PTHRESH; + rxdctl |= IGB_RX_HTHRESH << 8; + rxdctl |= IGB_RX_WTHRESH << 16; + E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); + } + } else if (adapter->hw.mac.type >= e1000_pch2lan) { + if (if_getmtu(ifp) > ETHERMTU) + e1000_lv_jumbo_workaround_ich8lan(hw, TRUE); + else + e1000_lv_jumbo_workaround_ich8lan(hw, FALSE); } - /* Catch any remaining refresh work */ - if (e1000_rx_unrefreshed(rxr)) - em_refresh_mbufs(rxr, i); - - rxr->next_to_check = i; - if (done != NULL) - *done = rxdone; - EM_RX_UNLOCK(rxr); - - return ((status & E1000_RXD_STAT_DD) ? TRUE : FALSE); -} - -static __inline void -em_rx_discard(struct rx_ring *rxr, int i) -{ - struct em_rxbuffer *rbuf; + /* Make sure VLAN Filters are off */ + rctl &= ~E1000_RCTL_VFE; - rbuf = &rxr->rx_buffers[i]; - bus_dmamap_unload(rxr->rxtag, rbuf->map); + if (adapter->hw.mac.type < igb_mac_min) { + if (adapter->rx_mbuf_sz == MCLBYTES) + rctl |= E1000_RCTL_SZ_2048; + else if (adapter->rx_mbuf_sz == MJUMPAGESIZE) + rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX; + else if (adapter->rx_mbuf_sz > MJUMPAGESIZE) + rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX; - /* Free any previous pieces */ - if (rxr->fmp != NULL) { - rxr->fmp->m_flags |= M_PKTHDR; - m_freem(rxr->fmp); - rxr->fmp = NULL; - rxr->lmp = NULL; - } - /* - ** Free buffer and allow em_refresh_mbufs() - ** to clean up and recharge buffer. - */ - if (rbuf->m_head) { - m_free(rbuf->m_head); - rbuf->m_head = NULL; - } - return; -} - -#ifndef __NO_STRICT_ALIGNMENT -/* - * When jumbo frames are enabled we should realign entire payload on - * architecures with strict alignment. This is serious design mistake of 8254x - * as it nullifies DMA operations. 8254x just allows RX buffer size to be - * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its - * payload. On architecures without strict alignment restrictions 8254x still - * performs unaligned memory access which would reduce the performance too. - * To avoid copying over an entire frame to align, we allocate a new mbuf and - * copy ethernet header to the new mbuf. The new mbuf is prepended into the - * existing mbuf chain. - * - * Be aware, best performance of the 8254x is achived only when jumbo frame is - * not used at all on architectures with strict alignment. - */ -static int -em_fixup_rx(struct rx_ring *rxr) -{ - struct adapter *adapter = rxr->adapter; - struct mbuf *m, *n; - int error; - - error = 0; - m = rxr->fmp; - if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) { - bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len); - m->m_data += ETHER_HDR_LEN; - } else { - MGETHDR(n, M_NOWAIT, MT_DATA); - if (n != NULL) { - bcopy(m->m_data, n->m_data, ETHER_HDR_LEN); - m->m_data += ETHER_HDR_LEN; - m->m_len -= ETHER_HDR_LEN; - n->m_len = ETHER_HDR_LEN; - M_MOVE_PKTHDR(n, m); - n->m_next = m; - rxr->fmp = n; - } else { - adapter->dropped_pkts++; - m_freem(rxr->fmp); - rxr->fmp = NULL; - error = ENOMEM; - } + /* ensure we clear use DTYPE of 00 here */ + rctl &= ~0x00000C00; } - return (error); -} -#endif - -static void -em_setup_rxdesc(union e1000_rx_desc_extended *rxd, const struct em_rxbuffer *rxbuf) -{ - rxd->read.buffer_addr = htole64(rxbuf->paddr); - /* DD bits must be cleared */ - rxd->wb.upper.status_error= 0; -} - -/********************************************************************* - * - * Verify that the hardware indicated that the checksum is valid. - * Inform the stack about the status of checksum so that stack - * doesn't spend time verifying the checksum. - * - *********************************************************************/ -static void -em_receive_checksum(uint32_t status, struct mbuf *mp) -{ - mp->m_pkthdr.csum_flags = 0; - - /* Ignore Checksum bit is set */ - if (status & E1000_RXD_STAT_IXSM) - return; - - /* If the IP checksum exists and there is no IP Checksum error */ - if ((status & (E1000_RXD_STAT_IPCS | E1000_RXDEXT_STATERR_IPE)) == - E1000_RXD_STAT_IPCS) { - mp->m_pkthdr.csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID); - } + /* Write out the settings */ + E1000_WRITE_REG(hw, E1000_RCTL, rctl); - /* TCP or UDP checksum */ - if ((status & (E1000_RXD_STAT_TCPCS | E1000_RXDEXT_STATERR_TCPE)) == - E1000_RXD_STAT_TCPCS) { - mp->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); - mp->m_pkthdr.csum_data = htons(0xffff); - } - if (status & E1000_RXD_STAT_UDPCS) { - mp->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); - mp->m_pkthdr.csum_data = htons(0xffff); - } + return; } -/* - * This routine is run via an vlan - * config EVENT - */ static void -em_register_vlan(void *arg, if_t ifp, u16 vtag) +em_if_vlan_register(if_ctx_t ctx, u16 vtag) { - struct adapter *adapter = if_getsoftc(ifp); - u32 index, bit; - - if ((void*)adapter != arg) /* Not our event */ - return; + struct adapter *adapter = iflib_get_softc(ctx); + u32 index, bit; - if ((vtag == 0) || (vtag > 4095)) /* Invalid ID */ - return; - - EM_CORE_LOCK(adapter); index = (vtag >> 5) & 0x7F; bit = vtag & 0x1F; adapter->shadow_vfta[index] |= (1 << bit); ++adapter->num_vlans; - /* Re-init to load the changes */ - if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) - em_init_locked(adapter); - EM_CORE_UNLOCK(adapter); } -/* - * This routine is run via an vlan - * unconfig EVENT - */ static void -em_unregister_vlan(void *arg, if_t ifp, u16 vtag) +em_if_vlan_unregister(if_ctx_t ctx, u16 vtag) { - struct adapter *adapter = if_getsoftc(ifp); - u32 index, bit; - - if (adapter != arg) - return; - - if ((vtag == 0) || (vtag > 4095)) /* Invalid */ - return; + struct adapter *adapter = iflib_get_softc(ctx); + u32 index, bit; - EM_CORE_LOCK(adapter); index = (vtag >> 5) & 0x7F; bit = vtag & 0x1F; adapter->shadow_vfta[index] &= ~(1 << bit); --adapter->num_vlans; - /* Re-init to load the changes */ - if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) - em_init_locked(adapter); - EM_CORE_UNLOCK(adapter); } static void em_setup_vlan_hw_support(struct adapter *adapter) { struct e1000_hw *hw = &adapter->hw; - u32 reg; + u32 reg; /* - ** We get here thru init_locked, meaning - ** a soft reset, this has already cleared - ** the VFTA and other state, so if there - ** have been no vlan's registered do nothing. - */ + * We get here thru init_locked, meaning + * a soft reset, this has already cleared + * the VFTA and other state, so if there + * have been no vlan's registered do nothing. + */ if (adapter->num_vlans == 0) - return; + return; /* - ** A soft reset zero's out the VFTA, so - ** we need to repopulate it now. - */ + * A soft reset zero's out the VFTA, so + * we need to repopulate it now. + */ for (int i = 0; i < EM_VFTA_SIZE; i++) - if (adapter->shadow_vfta[i] != 0) + if (adapter->shadow_vfta[i] != 0) E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, - i, adapter->shadow_vfta[i]); + i, adapter->shadow_vfta[i]); reg = E1000_READ_REG(hw, E1000_CTRL); reg |= E1000_CTRL_VME; @@ -5110,25 +3217,38 @@ em_setup_vlan_hw_support(struct adapter *adapter) } static void -em_enable_intr(struct adapter *adapter) +em_if_enable_intr(if_ctx_t ctx) { + struct adapter *adapter = iflib_get_softc(ctx); struct e1000_hw *hw = &adapter->hw; u32 ims_mask = IMS_ENABLE_MASK; if (hw->mac.type == e1000_82574) { - E1000_WRITE_REG(hw, EM_EIAC, adapter->ims); + E1000_WRITE_REG(hw, EM_EIAC, EM_MSIX_MASK); ims_mask |= adapter->ims; - } + } else if (adapter->intr_type == IFLIB_INTR_MSIX && hw->mac.type >= igb_mac_min) { + u32 mask = (adapter->que_mask | adapter->link_mask); + + E1000_WRITE_REG(&adapter->hw, E1000_EIAC, mask); + E1000_WRITE_REG(&adapter->hw, E1000_EIAM, mask); + E1000_WRITE_REG(&adapter->hw, E1000_EIMS, mask); + ims_mask = E1000_IMS_LSC; + } + E1000_WRITE_REG(hw, E1000_IMS, ims_mask); } static void -em_disable_intr(struct adapter *adapter) +em_if_disable_intr(if_ctx_t ctx) { + struct adapter *adapter = iflib_get_softc(ctx); struct e1000_hw *hw = &adapter->hw; - if (hw->mac.type == e1000_82574) - E1000_WRITE_REG(hw, EM_EIAC, 0); + if (adapter->intr_type == IFLIB_INTR_MSIX) { + if (hw->mac.type >= igb_mac_min) + E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0); + E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0); + } E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); } @@ -5149,7 +3269,7 @@ em_init_manageability(struct adapter *adapter) /* disable hardware interception of ARP */ manc &= ~(E1000_MANC_ARP_EN); - /* enable receiving management packets to the host */ + /* enable receiving management packets to the host */ manc |= E1000_MANC_EN_MNG2HOST; #define E1000_MNG2HOST_PORT_623 (1 << 5) #define E1000_MNG2HOST_PORT_664 (1 << 6) @@ -5247,19 +3367,38 @@ em_is_valid_ether_addr(u8 *addr) ** later use. */ static void -em_get_wakeup(device_t dev) +em_get_wakeup(if_ctx_t ctx) { - struct adapter *adapter = device_get_softc(dev); - u16 eeprom_data = 0, device_id, apme_mask; + struct adapter *adapter = iflib_get_softc(ctx); + device_t dev = iflib_get_dev(ctx); + u16 eeprom_data = 0, device_id, apme_mask; adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw); apme_mask = EM_EEPROM_APME; switch (adapter->hw.mac.type) { + case e1000_82542: + case e1000_82543: + break; + case e1000_82544: + e1000_read_nvm(&adapter->hw, + NVM_INIT_CONTROL2_REG, 1, &eeprom_data); + apme_mask = EM_82544_APME; + break; + case e1000_82546: + case e1000_82546_rev_3: + if (adapter->hw.bus.func == 1) { + e1000_read_nvm(&adapter->hw, + NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); + break; + } else + e1000_read_nvm(&adapter->hw, + NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); + break; case e1000_82573: case e1000_82583: adapter->has_amt = TRUE; - /* Falls thru */ + /* FALLTHROUGH */ case e1000_82571: case e1000_82572: case e1000_80003es2lan: @@ -5278,6 +3417,15 @@ em_get_wakeup(device_t dev) case e1000_pch2lan: case e1000_pch_lpt: case e1000_pch_spt: + case e1000_82575: /* listing all igb devices */ + case e1000_82576: + case e1000_82580: + case e1000_i350: + case e1000_i354: + case e1000_i210: + case e1000_i211: + case e1000_vfadapt: + case e1000_vfadapt_i350: apme_mask = E1000_WUC_APME; adapter->has_amt = TRUE; eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC); @@ -5290,12 +3438,31 @@ em_get_wakeup(device_t dev) if (eeprom_data & apme_mask) adapter->wol = (E1000_WUFC_MAG | E1000_WUFC_MC); /* - * We have the eeprom settings, now apply the special cases - * where the eeprom may be wrong or the board won't support - * wake on lan on a particular port + * We have the eeprom settings, now apply the special cases + * where the eeprom may be wrong or the board won't support + * wake on lan on a particular port */ device_id = pci_get_device(dev); - switch (device_id) { + switch (device_id) { + case E1000_DEV_ID_82546GB_PCIE: + adapter->wol = 0; + break; + case E1000_DEV_ID_82546EB_FIBER: + case E1000_DEV_ID_82546GB_FIBER: + /* Wake events only supported on port A for dual fiber + * regardless of eeprom setting */ + if (E1000_READ_REG(&adapter->hw, E1000_STATUS) & + E1000_STATUS_FUNC_1) + adapter->wol = 0; + break; + case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3: + /* if quad port adapter, disable WoL on all but port A */ + if (global_quad_port_a != 0) + adapter->wol = 0; + /* Reset for multiple quad port adapters */ + if (++global_quad_port_a == 4) + global_quad_port_a = 0; + break; case E1000_DEV_ID_82571EB_FIBER: /* Wake events only supported on port A for dual fiber * regardless of eeprom setting */ @@ -5306,13 +3473,13 @@ em_get_wakeup(device_t dev) case E1000_DEV_ID_82571EB_QUAD_COPPER: case E1000_DEV_ID_82571EB_QUAD_FIBER: case E1000_DEV_ID_82571EB_QUAD_COPPER_LP: - /* if quad port adapter, disable WoL on all but port A */ + /* if quad port adapter, disable WoL on all but port A */ if (global_quad_port_a != 0) adapter->wol = 0; /* Reset for multiple quad port adapters */ if (++global_quad_port_a == 4) global_quad_port_a = 0; - break; + break; } return; } @@ -5322,12 +3489,13 @@ em_get_wakeup(device_t dev) * Enable PCI Wake On Lan capability */ static void -em_enable_wakeup(device_t dev) +em_enable_wakeup(if_ctx_t ctx) { - struct adapter *adapter = device_get_softc(dev); - if_t ifp = adapter->ifp; - u32 pmc, ctrl, ctrl_ext, rctl, wuc; - u16 status; + struct adapter *adapter = iflib_get_softc(ctx); + device_t dev = iflib_get_dev(ctx); + if_t ifp = iflib_get_ifp(ctx); + u32 pmc, ctrl, ctrl_ext, rctl, wuc; + u16 status; if ((pci_find_cap(dev, PCIY_PMG, &pmc) != 0)) return; @@ -5337,8 +3505,8 @@ em_enable_wakeup(device_t dev) ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3); E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); wuc = E1000_READ_REG(&adapter->hw, E1000_WUC); - wuc |= E1000_WUC_PME_EN; - E1000_WRITE_REG(&adapter->hw, E1000_WUC, wuc); + wuc |= (E1000_WUC_PME_EN | E1000_WUC_APME); + E1000_WRITE_REG(&adapter->hw, E1000_WUC, wuc); if ((adapter->hw.mac.type == e1000_ich8lan) || (adapter->hw.mac.type == e1000_pchlan) || @@ -5355,12 +3523,15 @@ em_enable_wakeup(device_t dev) } /* - ** Determine type of Wakeup: note that wol - ** is set with all bits on by default. - */ + * Determine type of Wakeup: note that wol + * is set with all bits on by default. + */ if ((if_getcapenable(ifp) & IFCAP_WOL_MAGIC) == 0) adapter->wol &= ~E1000_WUFC_MAG; + if ((if_getcapenable(ifp) & IFCAP_WOL_UCAST) == 0) + adapter->wol &= ~E1000_WUFC_EX; + if ((if_getcapenable(ifp) & IFCAP_WOL_MCAST) == 0) adapter->wol &= ~E1000_WUFC_MC; else { @@ -5369,10 +3540,7 @@ em_enable_wakeup(device_t dev) E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); } - if ((adapter->hw.mac.type == e1000_pchlan) || - (adapter->hw.mac.type == e1000_pch2lan) || - (adapter->hw.mac.type == e1000_pch_lpt) || - (adapter->hw.mac.type == e1000_pch_spt)) { + if ( adapter->hw.mac.type >= e1000_pchlan) { if (em_enable_phy_wakeup(adapter)) return; } else { @@ -5383,20 +3551,20 @@ em_enable_wakeup(device_t dev) if (adapter->hw.phy.type == e1000_phy_igp_3) e1000_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw); - /* Request PME */ - status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2); + /* Request PME */ + status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2); status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE); if (if_getcapenable(ifp) & IFCAP_WOL) status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; - pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2); + pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2); return; } /* -** WOL in the newer chipset interfaces (pchlan) -** require thing to be copied into the phy -*/ + * WOL in the newer chipset interfaces (pchlan) + * require thing to be copied into the phy + */ static int em_enable_phy_wakeup(struct adapter *adapter) { @@ -5437,7 +3605,7 @@ em_enable_phy_wakeup(struct adapter *adapter) /* enable PHY wakeup in MAC register */ E1000_WRITE_REG(hw, E1000_WUC, - E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN); + E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN | E1000_WUC_APME); E1000_WRITE_REG(hw, E1000_WUFC, adapter->wol); /* configure and enable PHY wakeup in PHY registers */ @@ -5468,11 +3636,10 @@ out: } static void -em_led_func(void *arg, int onoff) +em_if_led_func(if_ctx_t ctx, int onoff) { - struct adapter *adapter = arg; + struct adapter *adapter = iflib_get_softc(ctx); - EM_CORE_LOCK(adapter); if (onoff) { e1000_setup_led(&adapter->hw); e1000_led_on(&adapter->hw); @@ -5480,26 +3647,25 @@ em_led_func(void *arg, int onoff) e1000_led_off(&adapter->hw); e1000_cleanup_led(&adapter->hw); } - EM_CORE_UNLOCK(adapter); } /* -** Disable the L0S and L1 LINK states -*/ + * Disable the L0S and L1 LINK states + */ static void em_disable_aspm(struct adapter *adapter) { - int base, reg; - u16 link_cap,link_ctrl; - device_t dev = adapter->dev; + int base, reg; + u16 link_cap,link_ctrl; + device_t dev = adapter->dev; switch (adapter->hw.mac.type) { - case e1000_82573: - case e1000_82574: - case e1000_82583: - break; - default: - return; + case e1000_82573: + case e1000_82574: + case e1000_82583: + break; + default: + return; } if (pci_find_cap(dev, PCIY_EXPRESS, &base) != 0) return; @@ -5595,27 +3761,26 @@ em_update_stats_counters(struct adapter *adapter) adapter->stats.icrxoc += E1000_READ_REG(&adapter->hw, E1000_ICRXOC); if (adapter->hw.mac.type >= e1000_82543) { - adapter->stats.algnerrc += + adapter->stats.algnerrc += E1000_READ_REG(&adapter->hw, E1000_ALGNERRC); - adapter->stats.rxerrc += + adapter->stats.rxerrc += E1000_READ_REG(&adapter->hw, E1000_RXERRC); - adapter->stats.tncrs += + adapter->stats.tncrs += E1000_READ_REG(&adapter->hw, E1000_TNCRS); - adapter->stats.cexterr += + adapter->stats.cexterr += E1000_READ_REG(&adapter->hw, E1000_CEXTERR); - adapter->stats.tsctc += + adapter->stats.tsctc += E1000_READ_REG(&adapter->hw, E1000_TSCTC); - adapter->stats.tsctfc += + adapter->stats.tsctfc += E1000_READ_REG(&adapter->hw, E1000_TSCTFC); } } static uint64_t -em_get_counter(if_t ifp, ift_counter cnt) +em_if_get_counter(if_ctx_t ctx, ift_counter cnt) { - struct adapter *adapter; - - adapter = if_getsoftc(ifp); + struct adapter *adapter = iflib_get_softc(ctx); + struct ifnet *ifp = iflib_get_ifp(ctx); switch (cnt) { case IFCOUNTER_COLLISIONS: @@ -5651,10 +3816,9 @@ em_sysctl_reg_handler(SYSCTL_HANDLER_ARGS) static void em_add_hw_stats(struct adapter *adapter) { - device_t dev = adapter->dev; - - struct tx_ring *txr = adapter->tx_rings; - struct rx_ring *rxr = adapter->rx_rings; + device_t dev = iflib_get_dev(adapter->ctx); + struct em_tx_queue *tx_que = adapter->tx_queues; + struct em_rx_queue *rx_que = adapter->rx_queues; struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); struct sysctl_oid *tree = device_get_sysctl_tree(dev); @@ -5666,18 +3830,18 @@ em_add_hw_stats(struct adapter *adapter) #define QUEUE_NAME_LEN 32 char namebuf[QUEUE_NAME_LEN]; - + /* Driver Statistics */ - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", + SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", CTLFLAG_RD, &adapter->dropped_pkts, "Driver dropped packets"); SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq", CTLFLAG_RD, &adapter->link_irq, "Link MSIX IRQ Handled"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_fail", + SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_fail", CTLFLAG_RD, &adapter->mbuf_defrag_failed, "Defragmenting mbuf chain failed"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail", + SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail", CTLFLAG_RD, &adapter->no_tx_dma_setup, "Driver tx dma failure in xmit"); SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns", @@ -5686,7 +3850,6 @@ em_add_hw_stats(struct adapter *adapter) SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts", CTLFLAG_RD, &adapter->watchdog_events, "Watchdog timeouts"); - SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control", CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_CTRL, em_sysctl_reg_handler, "IU", @@ -5698,44 +3861,48 @@ em_add_hw_stats(struct adapter *adapter) SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water", CTLFLAG_RD, &adapter->hw.fc.high_water, 0, "Flow Control High Watermark"); - SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", + SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", CTLFLAG_RD, &adapter->hw.fc.low_water, 0, "Flow Control Low Watermark"); - for (int i = 0; i < adapter->num_queues; i++, txr++, rxr++) { + for (int i = 0; i < adapter->tx_num_queues; i++, tx_que++) { + struct tx_ring *txr = &tx_que->txr; snprintf(namebuf, QUEUE_NAME_LEN, "queue_tx_%d", i); queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, NULL, "TX Queue Name"); queue_list = SYSCTL_CHILDREN(queue_node); - SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head", + SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head", CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDH(txr->me), em_sysctl_reg_handler, "IU", - "Transmit Descriptor Head"); - SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail", + "Transmit Descriptor Head"); + SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail", CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDT(txr->me), em_sysctl_reg_handler, "IU", - "Transmit Descriptor Tail"); + "Transmit Descriptor Tail"); SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tx_irq", CTLFLAG_RD, &txr->tx_irq, "Queue MSI-X Transmit Interrupts"); - SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_desc_avail", + SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_desc_avail", CTLFLAG_RD, &txr->no_desc_avail, "Queue No Descriptor Available"); + } - snprintf(namebuf, QUEUE_NAME_LEN, "queue_rx_%d", i); + for (int j = 0; j < adapter->rx_num_queues; j++, rx_que++) { + struct rx_ring *rxr = &rx_que->rxr; + snprintf(namebuf, QUEUE_NAME_LEN, "queue_rx_%d", j); queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, NULL, "RX Queue Name"); queue_list = SYSCTL_CHILDREN(queue_node); - SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head", + SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head", CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDH(rxr->me), em_sysctl_reg_handler, "IU", "Receive Descriptor Head"); - SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail", + SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail", CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDT(rxr->me), em_sysctl_reg_handler, "IU", @@ -5747,7 +3914,7 @@ em_add_hw_stats(struct adapter *adapter) /* MAC stats get their own sub node */ - stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", + stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", CTLFLAG_RD, NULL, "Statistics"); stat_list = SYSCTL_CHILDREN(stat_node); @@ -5850,14 +4017,14 @@ em_add_hw_stats(struct adapter *adapter) SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522", CTLFLAG_RD, &adapter->stats.prc1522, "1023-1522 byte frames received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", - CTLFLAG_RD, &adapter->stats.gorc, - "Good Octets Received"); + SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", + CTLFLAG_RD, &adapter->stats.gorc, + "Good Octets Received"); /* Packet Transmission Stats */ - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", - CTLFLAG_RD, &adapter->stats.gotc, - "Good Octets Transmitted"); + SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", + CTLFLAG_RD, &adapter->stats.gotc, + "Good Octets Transmitted"); SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd", CTLFLAG_RD, &adapter->stats.tpt, "Total Packets Transmitted"); @@ -5973,8 +4140,8 @@ em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS) static void em_print_nvm_info(struct adapter *adapter) { - u16 eeprom_data; - int i, j, row = 0; + u16 eeprom_data; + int i, j, row = 0; /* Its a bit crude, but it gets the job done */ printf("\nInterface EEPROM Dump:\n"); @@ -5998,7 +4165,7 @@ em_sysctl_int_delay(SYSCTL_HANDLER_ARGS) u32 regval; int error, usecs, ticks; - info = (struct em_int_delay_info *)arg1; + info = (struct em_int_delay_info *) arg1; usecs = info->value; error = sysctl_handle_int(oidp, &usecs, 0, req); if (error != 0 || req->newptr == NULL) @@ -6011,8 +4178,7 @@ em_sysctl_int_delay(SYSCTL_HANDLER_ARGS) ticks *= 4; adapter = info->adapter; - - EM_CORE_LOCK(adapter); + regval = E1000_READ_OFFSET(&adapter->hw, info->offset); regval = (regval & ~0xffff) | (ticks & 0xffff); /* Handle a few special cases. */ @@ -6029,7 +4195,6 @@ em_sysctl_int_delay(SYSCTL_HANDLER_ARGS) break; } E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval); - EM_CORE_UNLOCK(adapter); return (0); } @@ -6047,78 +4212,66 @@ em_add_int_delay_sysctl(struct adapter *adapter, const char *name, info, 0, em_sysctl_int_delay, "I", description); } -static void -em_set_sysctl_value(struct adapter *adapter, const char *name, - const char *description, int *limit, int value) -{ - *limit = value; - SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), - OID_AUTO, name, CTLFLAG_RW, limit, value, description); -} - - /* -** Set flow control using sysctl: -** Flow control values: -** 0 - off -** 1 - rx pause -** 2 - tx pause -** 3 - full -*/ + * Set flow control using sysctl: + * Flow control values: + * 0 - off + * 1 - rx pause + * 2 - tx pause + * 3 - full + */ static int em_set_flowcntl(SYSCTL_HANDLER_ARGS) -{ - int error; - static int input = 3; /* default is full */ - struct adapter *adapter = (struct adapter *) arg1; - - error = sysctl_handle_int(oidp, &input, 0, req); - - if ((error) || (req->newptr == NULL)) - return (error); - +{ + int error; + static int input = 3; /* default is full */ + struct adapter *adapter = (struct adapter *) arg1; + + error = sysctl_handle_int(oidp, &input, 0, req); + + if ((error) || (req->newptr == NULL)) + return (error); + if (input == adapter->fc) /* no change? */ return (error); - switch (input) { - case e1000_fc_rx_pause: - case e1000_fc_tx_pause: - case e1000_fc_full: - case e1000_fc_none: - adapter->hw.fc.requested_mode = input; - adapter->fc = input; - break; - default: - /* Do nothing */ - return (error); - } + switch (input) { + case e1000_fc_rx_pause: + case e1000_fc_tx_pause: + case e1000_fc_full: + case e1000_fc_none: + adapter->hw.fc.requested_mode = input; + adapter->fc = input; + break; + default: + /* Do nothing */ + return (error); + } - adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode; - e1000_force_mac_fc(&adapter->hw); - return (error); + adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode; + e1000_force_mac_fc(&adapter->hw); + return (error); } /* -** Manage Energy Efficient Ethernet: -** Control values: -** 0/1 - enabled/disabled -*/ + * Manage Energy Efficient Ethernet: + * Control values: + * 0/1 - enabled/disabled + */ static int em_sysctl_eee(SYSCTL_HANDLER_ARGS) { - struct adapter *adapter = (struct adapter *) arg1; - int error, value; - - value = adapter->hw.dev_spec.ich8lan.eee_disable; - error = sysctl_handle_int(oidp, &value, 0, req); - if (error || req->newptr == NULL) - return (error); - EM_CORE_LOCK(adapter); - adapter->hw.dev_spec.ich8lan.eee_disable = (value != 0); - em_init_locked(adapter); - EM_CORE_UNLOCK(adapter); - return (0); + struct adapter *adapter = (struct adapter *) arg1; + int error, value; + + value = adapter->hw.dev_spec.ich8lan.eee_disable; + error = sysctl_handle_int(oidp, &value, 0, req); + if (error || req->newptr == NULL) + return (error); + adapter->hw.dev_spec.ich8lan.eee_disable = (value != 0); + em_if_init(adapter->ctx); + + return (0); } static int @@ -6135,66 +4288,84 @@ em_sysctl_debug_info(SYSCTL_HANDLER_ARGS) return (error); if (result == 1) { - adapter = (struct adapter *)arg1; + adapter = (struct adapter *) arg1; em_print_debug_info(adapter); } return (error); } +static int +em_get_rs(SYSCTL_HANDLER_ARGS) +{ + struct adapter *adapter = (struct adapter *) arg1; + int error; + int result; + + result = 0; + error = sysctl_handle_int(oidp, &result, 0, req); + + if (error || !req->newptr || result != 1) + return (error); + em_dump_rs(adapter); + + return (error); +} + +static void +em_if_debug(if_ctx_t ctx) +{ + em_dump_rs(iflib_get_softc(ctx)); +} + /* -** This routine is meant to be fluid, add whatever is -** needed for debugging a problem. -jfv -*/ + * This routine is meant to be fluid, add whatever is + * needed for debugging a problem. -jfv + */ static void em_print_debug_info(struct adapter *adapter) { - device_t dev = adapter->dev; - struct tx_ring *txr = adapter->tx_rings; - struct rx_ring *rxr = adapter->rx_rings; + device_t dev = iflib_get_dev(adapter->ctx); + struct ifnet *ifp = iflib_get_ifp(adapter->ctx); + struct tx_ring *txr = &adapter->tx_queues->txr; + struct rx_ring *rxr = &adapter->rx_queues->rxr; - if (if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) printf("Interface is RUNNING "); else printf("Interface is NOT RUNNING\n"); - if (if_getdrvflags(adapter->ifp) & IFF_DRV_OACTIVE) + if (if_getdrvflags(ifp) & IFF_DRV_OACTIVE) printf("and INACTIVE\n"); else printf("and ACTIVE\n"); - for (int i = 0; i < adapter->num_queues; i++, txr++, rxr++) { + for (int i = 0; i < adapter->tx_num_queues; i++, txr++) { device_printf(dev, "TX Queue %d ------\n", i); device_printf(dev, "hw tdh = %d, hw tdt = %d\n", - E1000_READ_REG(&adapter->hw, E1000_TDH(i)), - E1000_READ_REG(&adapter->hw, E1000_TDT(i))); - device_printf(dev, "Tx Queue Status = %d\n", txr->busy); - device_printf(dev, "TX descriptors avail = %d\n", - txr->tx_avail); - device_printf(dev, "Tx Descriptors avail failure = %ld\n", - txr->no_desc_avail); - device_printf(dev, "RX Queue %d ------\n", i); + E1000_READ_REG(&adapter->hw, E1000_TDH(i)), + E1000_READ_REG(&adapter->hw, E1000_TDT(i))); + + } + for (int j=0; j < adapter->rx_num_queues; j++, rxr++) { + device_printf(dev, "RX Queue %d ------\n", j); device_printf(dev, "hw rdh = %d, hw rdt = %d\n", - E1000_READ_REG(&adapter->hw, E1000_RDH(i)), - E1000_READ_REG(&adapter->hw, E1000_RDT(i))); - device_printf(dev, "RX discarded packets = %ld\n", - rxr->rx_discarded); - device_printf(dev, "RX Next to Check = %d\n", rxr->next_to_check); - device_printf(dev, "RX Next to Refresh = %d\n", rxr->next_to_refresh); + E1000_READ_REG(&adapter->hw, E1000_RDH(j)), + E1000_READ_REG(&adapter->hw, E1000_RDT(j))); } } -#ifdef EM_MULTIQUEUE /* * 82574 only: * Write a new value to the EEPROM increasing the number of MSIX * vectors from 3 to 5, for proper multiqueue support. */ static void -em_enable_vectors_82574(struct adapter *adapter) +em_enable_vectors_82574(if_ctx_t ctx) { + struct adapter *adapter = iflib_get_softc(ctx); struct e1000_hw *hw = &adapter->hw; - device_t dev = adapter->dev; + device_t dev = iflib_get_dev(ctx); u16 edata; e1000_read_nvm(hw, EM_NVM_PCIE_CTRL, 1, &edata); @@ -6209,42 +4380,3 @@ em_enable_vectors_82574(struct adapter *adapter) device_printf(dev, "Writing to eeprom: done\n"); } } -#endif - -#ifdef DDB -DB_COMMAND(em_reset_dev, em_ddb_reset_dev) -{ - devclass_t dc; - int max_em; - - dc = devclass_find("em"); - max_em = devclass_get_maxunit(dc); - - for (int index = 0; index < (max_em - 1); index++) { - device_t dev; - dev = devclass_get_device(dc, index); - if (device_get_driver(dev) == &em_driver) { - struct adapter *adapter = device_get_softc(dev); - EM_CORE_LOCK(adapter); - em_init_locked(adapter); - EM_CORE_UNLOCK(adapter); - } - } -} -DB_COMMAND(em_dump_queue, em_ddb_dump_queue) -{ - devclass_t dc; - int max_em; - - dc = devclass_find("em"); - max_em = devclass_get_maxunit(dc); - - for (int index = 0; index < (max_em - 1); index++) { - device_t dev; - dev = devclass_get_device(dc, index); - if (device_get_driver(dev) == &em_driver) - em_print_debug_info(device_get_softc(dev)); - } - -} -#endif |