diff options
Diffstat (limited to 'freebsd/sys/dev')
78 files changed, 6553 insertions, 17524 deletions
diff --git a/freebsd/sys/dev/bce/if_bce.c b/freebsd/sys/dev/bce/if_bce.c index 26dacd7b..079e903d 100644 --- a/freebsd/sys/dev/bce/if_bce.c +++ b/freebsd/sys/dev/bce/if_bce.c @@ -2802,7 +2802,7 @@ bce_nvram_write(struct bce_softc *sc, u32 offset, u8 *data_buf, if (align_start || align_end) { buf = malloc(len32, M_DEVBUF, M_NOWAIT); - if (buf == 0) { + if (buf == NULL) { rc = ENOMEM; goto bce_nvram_write_exit; } diff --git a/freebsd/sys/dev/e1000/e1000_82575.c b/freebsd/sys/dev/e1000/e1000_82575.c index 83116e8e..ebf8371c 100644 --- a/freebsd/sys/dev/e1000/e1000_82575.c +++ b/freebsd/sys/dev/e1000/e1000_82575.c @@ -103,7 +103,6 @@ static s32 e1000_validate_nvm_checksum_with_offset(struct e1000_hw *hw, u16 offset); static s32 e1000_validate_nvm_checksum_i350(struct e1000_hw *hw); static s32 e1000_update_nvm_checksum_i350(struct e1000_hw *hw); -static void e1000_write_vfta_i350(struct e1000_hw *hw, u32 offset, u32 value); static void e1000_clear_vfta_i350(struct e1000_hw *hw); static void e1000_i2c_start(struct e1000_hw *hw); diff --git a/freebsd/sys/dev/e1000/e1000_82575.h b/freebsd/sys/dev/e1000/e1000_82575.h index 45fe132e..f8179560 100644 --- a/freebsd/sys/dev/e1000/e1000_82575.h +++ b/freebsd/sys/dev/e1000/e1000_82575.h @@ -493,6 +493,7 @@ enum e1000_promisc_type { void e1000_vfta_set_vf(struct e1000_hw *, u16, bool); void e1000_rlpml_set_vf(struct e1000_hw *, u16); s32 e1000_promisc_set_vf(struct e1000_hw *, enum e1000_promisc_type type); +void e1000_write_vfta_i350(struct e1000_hw *hw, u32 offset, u32 value); u16 e1000_rxpbs_adjust_82580(u32 data); s32 e1000_read_emi_reg(struct e1000_hw *hw, u16 addr, u16 *data); s32 e1000_set_eee_i350(struct e1000_hw *hw, bool adv1G, bool adv100M); diff --git a/freebsd/sys/dev/e1000/e1000_defines.h b/freebsd/sys/dev/e1000/e1000_defines.h index e33fe0fb..4c2b0903 100644 --- a/freebsd/sys/dev/e1000/e1000_defines.h +++ b/freebsd/sys/dev/e1000/e1000_defines.h @@ -469,6 +469,8 @@ #define ETHERNET_FCS_SIZE 4 #define MAX_JUMBO_FRAME_SIZE 0x3F00 +/* The datasheet maximum supported RX size is 9.5KB (9728 bytes) */ +#define MAX_RX_JUMBO_FRAME_SIZE 0x2600 #define E1000_TX_PTR_GAP 0x1F /* Extended Configuration Control and Size */ diff --git a/freebsd/sys/dev/e1000/e1000_ich8lan.c b/freebsd/sys/dev/e1000/e1000_ich8lan.c index 007488b2..6f6cb582 100644 --- a/freebsd/sys/dev/e1000/e1000_ich8lan.c +++ b/freebsd/sys/dev/e1000/e1000_ich8lan.c @@ -245,8 +245,7 @@ static bool e1000_phy_is_accessible_pchlan(struct e1000_hw *hw) if (ret_val) return FALSE; out: - if ((hw->mac.type == e1000_pch_lpt) || - (hw->mac.type == e1000_pch_spt)) { + if (hw->mac.type >= e1000_pch_lpt) { /* Only unforce SMBus if ME is not active */ if (!(E1000_READ_REG(hw, E1000_FWSM) & E1000_ICH_FWSM_FW_VALID)) { @@ -643,7 +642,7 @@ static s32 e1000_init_nvm_params_ich8lan(struct e1000_hw *hw) nvm->type = e1000_nvm_flash_sw; - if (hw->mac.type == e1000_pch_spt) { + if (hw->mac.type >= e1000_pch_spt) { /* in SPT, gfpreg doesn't exist. NVM size is taken from the * STRAP register. This is because in SPT the GbE Flash region * is no longer accessed through the flash registers. Instead, @@ -703,7 +702,7 @@ static s32 e1000_init_nvm_params_ich8lan(struct e1000_hw *hw) /* Function Pointers */ nvm->ops.acquire = e1000_acquire_nvm_ich8lan; nvm->ops.release = e1000_release_nvm_ich8lan; - if (hw->mac.type == e1000_pch_spt) { + if (hw->mac.type >= e1000_pch_spt) { nvm->ops.read = e1000_read_nvm_spt; nvm->ops.update = e1000_update_nvm_checksum_spt; } else { @@ -817,8 +816,7 @@ static s32 e1000_init_mac_params_ich8lan(struct e1000_hw *hw) break; } - if ((mac->type == e1000_pch_lpt) || - (mac->type == e1000_pch_spt)) { + if (mac->type >= e1000_pch_lpt) { mac->rar_entry_count = E1000_PCH_LPT_RAR_ENTRIES; mac->ops.rar_set = e1000_rar_set_pch_lpt; mac->ops.setup_physical_interface = e1000_setup_copper_link_pch_lpt; @@ -1578,9 +1576,7 @@ static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw) * aggressive resulting in many collisions. To avoid this, increase * the IPG and reduce Rx latency in the PHY. */ - if (((hw->mac.type == e1000_pch2lan) || - (hw->mac.type == e1000_pch_lpt) || - (hw->mac.type == e1000_pch_spt)) && link) { + if ((hw->mac.type >= e1000_pch2lan) && link) { u16 speed, duplex; e1000_get_speed_and_duplex_copper_generic(hw, &speed, &duplex); @@ -1591,7 +1587,7 @@ static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw) tipg_reg |= 0xFF; /* Reduce Rx latency in analog PHY */ emi_val = 0; - } else if (hw->mac.type == e1000_pch_spt && + } else if (hw->mac.type >= e1000_pch_spt && duplex == FULL_DUPLEX && speed != SPEED_1000) { tipg_reg |= 0xC; emi_val = 1; @@ -1613,8 +1609,8 @@ static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw) emi_addr = I217_RX_CONFIG; ret_val = e1000_write_emi_reg_locked(hw, emi_addr, emi_val); - if (hw->mac.type == e1000_pch_lpt || - hw->mac.type == e1000_pch_spt) { + + if (hw->mac.type >= e1000_pch_lpt) { u16 phy_reg; hw->phy.ops.read_reg_locked(hw, I217_PLL_CLOCK_GATE_REG, @@ -1643,7 +1639,7 @@ static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw) if (ret_val) return ret_val; - if (hw->mac.type == e1000_pch_spt) { + if (hw->mac.type >= e1000_pch_spt) { u16 data; u16 ptr_gap; @@ -1692,8 +1688,7 @@ static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw) * on power up. * Set the Beacon Duration for I217 to 8 usec */ - if ((hw->mac.type == e1000_pch_lpt) || - (hw->mac.type == e1000_pch_spt)) { + if (hw->mac.type >= e1000_pch_lpt) { u32 mac_reg; mac_reg = E1000_READ_REG(hw, E1000_FEXTNVM4); @@ -1711,8 +1706,7 @@ static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw) if (ret_val) return ret_val; } - if ((hw->mac.type == e1000_pch_lpt) || - (hw->mac.type == e1000_pch_spt)) { + if (hw->mac.type >= e1000_pch_lpt) { /* Set platform power management values for * Latency Tolerance Reporting (LTR) * Optimized Buffer Flush/Fill (OBFF) @@ -1725,15 +1719,20 @@ static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw) /* Clear link partner's EEE ability */ hw->dev_spec.ich8lan.eee_lp_ability = 0; - /* FEXTNVM6 K1-off workaround */ - if (hw->mac.type == e1000_pch_spt) { - u32 pcieanacfg = E1000_READ_REG(hw, E1000_PCIEANACFG); + if (hw->mac.type >= e1000_pch_lpt) { u32 fextnvm6 = E1000_READ_REG(hw, E1000_FEXTNVM6); - if ((pcieanacfg & E1000_FEXTNVM6_K1_OFF_ENABLE) && - (hw->dev_spec.ich8lan.disable_k1_off == FALSE)) - fextnvm6 |= E1000_FEXTNVM6_K1_OFF_ENABLE; - else + if (hw->mac.type == e1000_pch_spt) { + /* FEXTNVM6 K1-off workaround - for SPT only */ + u32 pcieanacfg = E1000_READ_REG(hw, E1000_PCIEANACFG); + + if (pcieanacfg & E1000_FEXTNVM6_K1_OFF_ENABLE) + fextnvm6 |= E1000_FEXTNVM6_K1_OFF_ENABLE; + else + fextnvm6 &= ~E1000_FEXTNVM6_K1_OFF_ENABLE; + } + + if (hw->dev_spec.ich8lan.disable_k1_off == TRUE) fextnvm6 &= ~E1000_FEXTNVM6_K1_OFF_ENABLE; E1000_WRITE_REG(hw, E1000_FEXTNVM6, fextnvm6); @@ -3673,7 +3672,7 @@ static s32 e1000_flash_cycle_init_ich8lan(struct e1000_hw *hw) /* Clear FCERR and DAEL in hw status by writing 1 */ hsfsts.hsf_status.flcerr = 1; hsfsts.hsf_status.dael = 1; - if (hw->mac.type == e1000_pch_spt) + if (hw->mac.type >= e1000_pch_spt) E1000_WRITE_FLASH_REG(hw, ICH_FLASH_HSFSTS, hsfsts.regval & 0xFFFF); else @@ -3693,7 +3692,7 @@ static s32 e1000_flash_cycle_init_ich8lan(struct e1000_hw *hw) * Begin by setting Flash Cycle Done. */ hsfsts.hsf_status.flcdone = 1; - if (hw->mac.type == e1000_pch_spt) + if (hw->mac.type >= e1000_pch_spt) E1000_WRITE_FLASH_REG(hw, ICH_FLASH_HSFSTS, hsfsts.regval & 0xFFFF); else @@ -3720,7 +3719,7 @@ static s32 e1000_flash_cycle_init_ich8lan(struct e1000_hw *hw) * now set the Flash Cycle Done. */ hsfsts.hsf_status.flcdone = 1; - if (hw->mac.type == e1000_pch_spt) + if (hw->mac.type >= e1000_pch_spt) E1000_WRITE_FLASH_REG(hw, ICH_FLASH_HSFSTS, hsfsts.regval & 0xFFFF); else @@ -3750,13 +3749,13 @@ static s32 e1000_flash_cycle_ich8lan(struct e1000_hw *hw, u32 timeout) DEBUGFUNC("e1000_flash_cycle_ich8lan"); /* Start a cycle by writing 1 in Flash Cycle Go in Hw Flash Control */ - if (hw->mac.type == e1000_pch_spt) + if (hw->mac.type >= e1000_pch_spt) hsflctl.regval = E1000_READ_FLASH_REG(hw, ICH_FLASH_HSFSTS)>>16; else hsflctl.regval = E1000_READ_FLASH_REG16(hw, ICH_FLASH_HSFCTL); hsflctl.hsf_ctrl.flcgo = 1; - if (hw->mac.type == e1000_pch_spt) + if (hw->mac.type >= e1000_pch_spt) E1000_WRITE_FLASH_REG(hw, ICH_FLASH_HSFSTS, hsflctl.regval << 16); else @@ -3839,7 +3838,7 @@ static s32 e1000_read_flash_byte_ich8lan(struct e1000_hw *hw, u32 offset, /* In SPT, only 32 bits access is supported, * so this function should not be called. */ - if (hw->mac.type == e1000_pch_spt) + if (hw->mac.type >= e1000_pch_spt) return -E1000_ERR_NVM; else ret_val = e1000_read_flash_data_ich8lan(hw, offset, 1, &word); @@ -3948,7 +3947,7 @@ static s32 e1000_read_flash_data32_ich8lan(struct e1000_hw *hw, u32 offset, DEBUGFUNC("e1000_read_flash_data_ich8lan"); if (offset > ICH_FLASH_LINEAR_ADDR_MASK || - hw->mac.type != e1000_pch_spt) + hw->mac.type < e1000_pch_spt) return -E1000_ERR_NVM; flash_linear_addr = ((ICH_FLASH_LINEAR_ADDR_MASK & offset) + hw->nvm.flash_base_addr); @@ -4436,7 +4435,7 @@ static s32 e1000_write_flash_data_ich8lan(struct e1000_hw *hw, u32 offset, DEBUGFUNC("e1000_write_ich8_data"); - if (hw->mac.type == e1000_pch_spt) { + if (hw->mac.type >= e1000_pch_spt) { if (size != 4 || offset > ICH_FLASH_LINEAR_ADDR_MASK) return -E1000_ERR_NVM; } else { @@ -4456,7 +4455,7 @@ static s32 e1000_write_flash_data_ich8lan(struct e1000_hw *hw, u32 offset, /* In SPT, This register is in Lan memory space, not * flash. Therefore, only 32 bit access is supported */ - if (hw->mac.type == e1000_pch_spt) + if (hw->mac.type >= e1000_pch_spt) hsflctl.regval = E1000_READ_FLASH_REG(hw, ICH_FLASH_HSFSTS)>>16; else @@ -4470,7 +4469,7 @@ static s32 e1000_write_flash_data_ich8lan(struct e1000_hw *hw, u32 offset, * not flash. Therefore, only 32 bit access is * supported */ - if (hw->mac.type == e1000_pch_spt) + if (hw->mac.type >= e1000_pch_spt) E1000_WRITE_FLASH_REG(hw, ICH_FLASH_HSFSTS, hsflctl.regval << 16); else @@ -4532,7 +4531,7 @@ static s32 e1000_write_flash_data32_ich8lan(struct e1000_hw *hw, u32 offset, DEBUGFUNC("e1000_write_flash_data32_ich8lan"); - if (hw->mac.type == e1000_pch_spt) { + if (hw->mac.type >= e1000_pch_spt) { if (offset > ICH_FLASH_LINEAR_ADDR_MASK) return -E1000_ERR_NVM; } @@ -4548,7 +4547,7 @@ static s32 e1000_write_flash_data32_ich8lan(struct e1000_hw *hw, u32 offset, /* In SPT, This register is in Lan memory space, not * flash. Therefore, only 32 bit access is supported */ - if (hw->mac.type == e1000_pch_spt) + if (hw->mac.type >= e1000_pch_spt) hsflctl.regval = E1000_READ_FLASH_REG(hw, ICH_FLASH_HSFSTS) >> 16; @@ -4563,7 +4562,7 @@ static s32 e1000_write_flash_data32_ich8lan(struct e1000_hw *hw, u32 offset, * not flash. Therefore, only 32 bit access is * supported */ - if (hw->mac.type == e1000_pch_spt) + if (hw->mac.type >= e1000_pch_spt) E1000_WRITE_FLASH_REG(hw, ICH_FLASH_HSFSTS, hsflctl.regval << 16); else @@ -4765,7 +4764,7 @@ static s32 e1000_erase_flash_bank_ich8lan(struct e1000_hw *hw, u32 bank) /* Write a value 11 (block Erase) in Flash * Cycle field in hw flash control */ - if (hw->mac.type == e1000_pch_spt) + if (hw->mac.type >= e1000_pch_spt) hsflctl.regval = E1000_READ_FLASH_REG(hw, ICH_FLASH_HSFSTS)>>16; @@ -4775,7 +4774,7 @@ static s32 e1000_erase_flash_bank_ich8lan(struct e1000_hw *hw, u32 bank) ICH_FLASH_HSFCTL); hsflctl.hsf_ctrl.flcycle = ICH_CYCLE_ERASE; - if (hw->mac.type == e1000_pch_spt) + if (hw->mac.type >= e1000_pch_spt) E1000_WRITE_FLASH_REG(hw, ICH_FLASH_HSFSTS, hsflctl.regval << 16); else @@ -5213,8 +5212,7 @@ static void e1000_initialize_hw_bits_ich8lan(struct e1000_hw *hw) E1000_WRITE_REG(hw, E1000_RFCTL, reg); /* Enable ECC on Lynxpoint */ - if ((hw->mac.type == e1000_pch_lpt) || - (hw->mac.type == e1000_pch_spt)) { + if (hw->mac.type >= e1000_pch_lpt) { reg = E1000_READ_REG(hw, E1000_PBECCSTS); reg |= E1000_PBECCSTS_ECC_ENABLE; E1000_WRITE_REG(hw, E1000_PBECCSTS, reg); @@ -5647,7 +5645,7 @@ void e1000_suspend_workarounds_ich8lan(struct e1000_hw *hw) (device_id == E1000_DEV_ID_PCH_LPTLP_I218_V) || (device_id == E1000_DEV_ID_PCH_I218_LM3) || (device_id == E1000_DEV_ID_PCH_I218_V3) || - (hw->mac.type == e1000_pch_spt)) { + (hw->mac.type >= e1000_pch_spt)) { u32 fextnvm6 = E1000_READ_REG(hw, E1000_FEXTNVM6); E1000_WRITE_REG(hw, E1000_FEXTNVM6, diff --git a/freebsd/sys/dev/e1000/em_txrx.c b/freebsd/sys/dev/e1000/em_txrx.c new file mode 100644 index 00000000..2183a8bd --- /dev/null +++ b/freebsd/sys/dev/e1000/em_txrx.c @@ -0,0 +1,817 @@ +#include <machine/rtems-bsd-kernel-space.h> + +/*- + * Copyright (c) 2016-2017 Matt Macy <mmacy@nextbsd.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* $FreeBSD$ */ +#include "if_em.h" + +#ifdef RSS +#include <net/rss_config.h> +#include <netinet/in_rss.h> +#endif + +#ifdef VERBOSE_DEBUG +#define DPRINTF device_printf +#else +#define DPRINTF(...) +#endif + +/********************************************************************* + * Local Function prototypes + *********************************************************************/ +static int em_tso_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, + u32 *txd_lower); +static int em_transmit_checksum_setup(struct adapter *adapter, if_pkt_info_t pi, + u32 *txd_upper, u32 *txd_lower); +static int em_isc_txd_encap(void *arg, if_pkt_info_t pi); +static void em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx); +static int em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear); +static void em_isc_rxd_refill(void *arg, if_rxd_update_t iru); +static void em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, + qidx_t pidx); +static int em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, + qidx_t budget); +static int em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); + +static void lem_isc_rxd_refill(void *arg, if_rxd_update_t iru); + +static int lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, + qidx_t budget); +static int lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); + +static void lem_receive_checksum(int status, int errors, if_rxd_info_t ri); +static void em_receive_checksum(uint32_t status, if_rxd_info_t ri); +static int em_determine_rsstype(u32 pkt_info); +extern int em_intr(void *arg); + +struct if_txrx em_txrx = { + em_isc_txd_encap, + em_isc_txd_flush, + em_isc_txd_credits_update, + em_isc_rxd_available, + em_isc_rxd_pkt_get, + em_isc_rxd_refill, + em_isc_rxd_flush, + em_intr +}; + +struct if_txrx lem_txrx = { + em_isc_txd_encap, + em_isc_txd_flush, + em_isc_txd_credits_update, + lem_isc_rxd_available, + lem_isc_rxd_pkt_get, + lem_isc_rxd_refill, + em_isc_rxd_flush, + em_intr +}; + +extern if_shared_ctx_t em_sctx; + +void +em_dump_rs(struct adapter *adapter) +{ + if_softc_ctx_t scctx = adapter->shared; + struct em_tx_queue *que; + struct tx_ring *txr; + qidx_t i, ntxd, qid, cur; + int16_t rs_cidx; + uint8_t status; + + printf("\n"); + ntxd = scctx->isc_ntxd[0]; + for (qid = 0; qid < adapter->tx_num_queues; qid++) { + que = &adapter->tx_queues[qid]; + txr = &que->txr; + rs_cidx = txr->tx_rs_cidx; + if (rs_cidx != txr->tx_rs_pidx) { + cur = txr->tx_rsq[rs_cidx]; + status = txr->tx_base[cur].upper.fields.status; + if (!(status & E1000_TXD_STAT_DD)) + printf("qid[%d]->tx_rsq[%d]: %d clear ", qid, rs_cidx, cur); + } else { + rs_cidx = (rs_cidx-1)&(ntxd-1); + cur = txr->tx_rsq[rs_cidx]; + printf("qid[%d]->tx_rsq[rs_cidx-1=%d]: %d ", qid, rs_cidx, cur); + } + printf("cidx_prev=%d rs_pidx=%d ",txr->tx_cidx_processed, txr->tx_rs_pidx); + for (i = 0; i < ntxd; i++) { + if (txr->tx_base[i].upper.fields.status & E1000_TXD_STAT_DD) + printf("%d set ", i); + } + printf("\n"); + } +} + +/********************************************************************** + * + * Setup work for hardware segmentation offload (TSO) on + * adapters using advanced tx descriptors + * + **********************************************************************/ +static int +em_tso_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower) +{ + if_softc_ctx_t scctx = adapter->shared; + struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx]; + struct tx_ring *txr = &que->txr; + struct e1000_context_desc *TXD; + int cur, hdr_len; + + hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen; + *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ + E1000_TXD_DTYP_D | /* Data descr type */ + E1000_TXD_CMD_TSE); /* Do TSE on this packet */ + + /* IP and/or TCP header checksum calculation and insertion. */ + *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8; + + cur = pi->ipi_pidx; + TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; + + /* + * Start offset for header checksum calculation. + * End offset for header checksum calculation. + * Offset of place put the checksum. + */ + TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen; + TXD->lower_setup.ip_fields.ipcse = + htole16(pi->ipi_ehdrlen + pi->ipi_ip_hlen - 1); + TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen + offsetof(struct ip, ip_sum); + + /* + * Start offset for payload checksum calculation. + * End offset for payload checksum calculation. + * Offset of place to put the checksum. + */ + TXD->upper_setup.tcp_fields.tucss = pi->ipi_ehdrlen + pi->ipi_ip_hlen; + TXD->upper_setup.tcp_fields.tucse = 0; + TXD->upper_setup.tcp_fields.tucso = + pi->ipi_ehdrlen + pi->ipi_ip_hlen + offsetof(struct tcphdr, th_sum); + + /* + * Payload size per packet w/o any headers. + * Length of all headers up to payload. + */ + TXD->tcp_seg_setup.fields.mss = htole16(pi->ipi_tso_segsz); + TXD->tcp_seg_setup.fields.hdr_len = hdr_len; + + TXD->cmd_and_length = htole32(adapter->txd_cmd | + E1000_TXD_CMD_DEXT | /* Extended descr */ + E1000_TXD_CMD_TSE | /* TSE context */ + E1000_TXD_CMD_IP | /* Do IP csum */ + E1000_TXD_CMD_TCP | /* Do TCP checksum */ + (pi->ipi_len - hdr_len)); /* Total len */ + txr->tx_tso = TRUE; + + if (++cur == scctx->isc_ntxd[0]) { + cur = 0; + } + DPRINTF(iflib_get_dev(adapter->ctx), "%s: pidx: %d cur: %d\n", __FUNCTION__, pi->ipi_pidx, cur); + return (cur); +} + +#define TSO_WORKAROUND 4 +#define DONT_FORCE_CTX 1 + + +/********************************************************************* + * The offload context is protocol specific (TCP/UDP) and thus + * only needs to be set when the protocol changes. The occasion + * of a context change can be a performance detriment, and + * might be better just disabled. The reason arises in the way + * in which the controller supports pipelined requests from the + * Tx data DMA. Up to four requests can be pipelined, and they may + * belong to the same packet or to multiple packets. However all + * requests for one packet are issued before a request is issued + * for a subsequent packet and if a request for the next packet + * requires a context change, that request will be stalled + * until the previous request completes. This means setting up + * a new context effectively disables pipelined Tx data DMA which + * in turn greatly slow down performance to send small sized + * frames. + **********************************************************************/ + +static int +em_transmit_checksum_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower) +{ + struct e1000_context_desc *TXD = NULL; + if_softc_ctx_t scctx = adapter->shared; + struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx]; + struct tx_ring *txr = &que->txr; + int csum_flags = pi->ipi_csum_flags; + int cur, hdr_len; + u32 cmd; + + cur = pi->ipi_pidx; + hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen; + cmd = adapter->txd_cmd; + + /* + * The 82574L can only remember the *last* context used + * regardless of queue that it was use for. We cannot reuse + * contexts on this hardware platform and must generate a new + * context every time. 82574L hardware spec, section 7.2.6, + * second note. + */ + if (DONT_FORCE_CTX && + adapter->tx_num_queues == 1 && + txr->csum_lhlen == pi->ipi_ehdrlen && + txr->csum_iphlen == pi->ipi_ip_hlen && + txr->csum_flags == csum_flags) { + /* + * Same csum offload context as the previous packets; + * just return. + */ + *txd_upper = txr->csum_txd_upper; + *txd_lower = txr->csum_txd_lower; + return (cur); + } + + TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; + if (csum_flags & CSUM_IP) { + *txd_upper |= E1000_TXD_POPTS_IXSM << 8; + /* + * Start offset for header checksum calculation. + * End offset for header checksum calculation. + * Offset of place to put the checksum. + */ + TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen; + TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len); + TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen + offsetof(struct ip, ip_sum); + cmd |= E1000_TXD_CMD_IP; + } + + if (csum_flags & (CSUM_TCP|CSUM_UDP)) { + uint8_t tucso; + + *txd_upper |= E1000_TXD_POPTS_TXSM << 8; + *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; + + if (csum_flags & CSUM_TCP) { + tucso = hdr_len + offsetof(struct tcphdr, th_sum); + cmd |= E1000_TXD_CMD_TCP; + } else + tucso = hdr_len + offsetof(struct udphdr, uh_sum); + TXD->upper_setup.tcp_fields.tucss = hdr_len; + TXD->upper_setup.tcp_fields.tucse = htole16(0); + TXD->upper_setup.tcp_fields.tucso = tucso; + } + + txr->csum_lhlen = pi->ipi_ehdrlen; + txr->csum_iphlen = pi->ipi_ip_hlen; + txr->csum_flags = csum_flags; + txr->csum_txd_upper = *txd_upper; + txr->csum_txd_lower = *txd_lower; + + TXD->tcp_seg_setup.data = htole32(0); + TXD->cmd_and_length = + htole32(E1000_TXD_CMD_IFCS | E1000_TXD_CMD_DEXT | cmd); + + if (++cur == scctx->isc_ntxd[0]) { + cur = 0; + } + DPRINTF(iflib_get_dev(adapter->ctx), "checksum_setup csum_flags=%x txd_upper=%x txd_lower=%x hdr_len=%d cmd=%x\n", + csum_flags, *txd_upper, *txd_lower, hdr_len, cmd); + return (cur); +} + +static int +em_isc_txd_encap(void *arg, if_pkt_info_t pi) +{ + struct adapter *sc = arg; + if_softc_ctx_t scctx = sc->shared; + struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; + struct tx_ring *txr = &que->txr; + bus_dma_segment_t *segs = pi->ipi_segs; + int nsegs = pi->ipi_nsegs; + int csum_flags = pi->ipi_csum_flags; + int i, j, first, pidx_last; + u32 txd_flags, txd_upper = 0, txd_lower = 0; + + struct e1000_tx_desc *ctxd = NULL; + bool do_tso, tso_desc; + qidx_t ntxd; + + txd_flags = pi->ipi_flags & IPI_TX_INTR ? E1000_TXD_CMD_RS : 0; + i = first = pi->ipi_pidx; + do_tso = (csum_flags & CSUM_TSO); + tso_desc = FALSE; + ntxd = scctx->isc_ntxd[0]; + /* + * TSO Hardware workaround, if this packet is not + * TSO, and is only a single descriptor long, and + * it follows a TSO burst, then we need to add a + * sentinel descriptor to prevent premature writeback. + */ + if ((!do_tso) && (txr->tx_tso == TRUE)) { + if (nsegs == 1) + tso_desc = TRUE; + txr->tx_tso = FALSE; + } + + /* Do hardware assists */ + if (do_tso) { + i = em_tso_setup(sc, pi, &txd_upper, &txd_lower); + tso_desc = TRUE; + } else if (csum_flags & EM_CSUM_OFFLOAD) { + i = em_transmit_checksum_setup(sc, pi, &txd_upper, &txd_lower); + } + + if (pi->ipi_mflags & M_VLANTAG) { + /* Set the vlan id. */ + txd_upper |= htole16(pi->ipi_vtag) << 16; + /* Tell hardware to add tag */ + txd_lower |= htole32(E1000_TXD_CMD_VLE); + } + + DPRINTF(iflib_get_dev(sc->ctx), "encap: set up tx: nsegs=%d first=%d i=%d\n", nsegs, first, i); + /* XXX adapter->pcix_82544 -- lem_fill_descriptors */ + + /* Set up our transmit descriptors */ + for (j = 0; j < nsegs; j++) { + bus_size_t seg_len; + bus_addr_t seg_addr; + uint32_t cmd; + + ctxd = &txr->tx_base[i]; + seg_addr = segs[j].ds_addr; + seg_len = segs[j].ds_len; + cmd = E1000_TXD_CMD_IFCS | sc->txd_cmd; + + /* + * TSO Workaround: + * If this is the last descriptor, we want to + * split it so we have a small final sentinel + */ + if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) { + seg_len -= TSO_WORKAROUND; + ctxd->buffer_addr = htole64(seg_addr); + ctxd->lower.data = htole32(cmd | txd_lower | seg_len); + ctxd->upper.data = htole32(txd_upper); + + if (++i == scctx->isc_ntxd[0]) + i = 0; + + /* Now make the sentinel */ + ctxd = &txr->tx_base[i]; + ctxd->buffer_addr = htole64(seg_addr + seg_len); + ctxd->lower.data = htole32(cmd | txd_lower | TSO_WORKAROUND); + ctxd->upper.data = htole32(txd_upper); + pidx_last = i; + if (++i == scctx->isc_ntxd[0]) + i = 0; + DPRINTF(iflib_get_dev(sc->ctx), "TSO path pidx_last=%d i=%d ntxd[0]=%d\n", pidx_last, i, scctx->isc_ntxd[0]); + } else { + ctxd->buffer_addr = htole64(seg_addr); + ctxd->lower.data = htole32(cmd | txd_lower | seg_len); + ctxd->upper.data = htole32(txd_upper); + pidx_last = i; + if (++i == scctx->isc_ntxd[0]) + i = 0; + DPRINTF(iflib_get_dev(sc->ctx), "pidx_last=%d i=%d ntxd[0]=%d\n", pidx_last, i, scctx->isc_ntxd[0]); + } + } + + /* + * Last Descriptor of Packet + * needs End Of Packet (EOP) + * and Report Status (RS) + */ + if (txd_flags) { + txr->tx_rsq[txr->tx_rs_pidx] = pidx_last; + DPRINTF(iflib_get_dev(sc->ctx), "setting to RS on %d rs_pidx %d first: %d\n", pidx_last, txr->tx_rs_pidx, first); + txr->tx_rs_pidx = (txr->tx_rs_pidx+1) & (ntxd-1); + MPASS(txr->tx_rs_pidx != txr->tx_rs_cidx); + } + ctxd->lower.data |= htole32(E1000_TXD_CMD_EOP | txd_flags); + DPRINTF(iflib_get_dev(sc->ctx), "tx_buffers[%d]->eop = %d ipi_new_pidx=%d\n", first, pidx_last, i); + pi->ipi_new_pidx = i; + + return (0); +} + +static void +em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx) +{ + struct adapter *adapter = arg; + struct em_tx_queue *que = &adapter->tx_queues[txqid]; + struct tx_ring *txr = &que->txr; + + E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), pidx); +} + +static int +em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear) +{ + struct adapter *adapter = arg; + if_softc_ctx_t scctx = adapter->shared; + struct em_tx_queue *que = &adapter->tx_queues[txqid]; + struct tx_ring *txr = &que->txr; + + qidx_t processed = 0; + int updated; + qidx_t cur, prev, ntxd, rs_cidx; + int32_t delta; + uint8_t status; + + rs_cidx = txr->tx_rs_cidx; + if (rs_cidx == txr->tx_rs_pidx) + return (0); + cur = txr->tx_rsq[rs_cidx]; + MPASS(cur != QIDX_INVALID); + status = txr->tx_base[cur].upper.fields.status; + updated = !!(status & E1000_TXD_STAT_DD); + + if (clear == false || updated == 0) + return (updated); + + prev = txr->tx_cidx_processed; + ntxd = scctx->isc_ntxd[0]; + do { + delta = (int32_t)cur - (int32_t)prev; + MPASS(prev == 0 || delta != 0); + if (delta < 0) + delta += ntxd; + DPRINTF(iflib_get_dev(adapter->ctx), + "%s: cidx_processed=%u cur=%u clear=%d delta=%d\n", + __FUNCTION__, prev, cur, clear, delta); + + processed += delta; + prev = cur; + rs_cidx = (rs_cidx + 1) & (ntxd-1); + if (rs_cidx == txr->tx_rs_pidx) + break; + cur = txr->tx_rsq[rs_cidx]; + MPASS(cur != QIDX_INVALID); + status = txr->tx_base[cur].upper.fields.status; + } while ((status & E1000_TXD_STAT_DD)); + + txr->tx_rs_cidx = rs_cidx; + txr->tx_cidx_processed = prev; + return(processed); +} + +static void +lem_isc_rxd_refill(void *arg, if_rxd_update_t iru) +{ + struct adapter *sc = arg; + if_softc_ctx_t scctx = sc->shared; + struct em_rx_queue *que = &sc->rx_queues[iru->iru_qsidx]; + struct rx_ring *rxr = &que->rxr; + struct e1000_rx_desc *rxd; + uint64_t *paddrs; + uint32_t next_pidx, pidx; + uint16_t count; + int i; + + paddrs = iru->iru_paddrs; + pidx = iru->iru_pidx; + count = iru->iru_count; + + for (i = 0, next_pidx = pidx; i < count; i++) { + rxd = (struct e1000_rx_desc *)&rxr->rx_base[next_pidx]; + rxd->buffer_addr = htole64(paddrs[i]); + /* status bits must be cleared */ + rxd->status = 0; + + if (++next_pidx == scctx->isc_nrxd[0]) + next_pidx = 0; + } +} + +static void +em_isc_rxd_refill(void *arg, if_rxd_update_t iru) +{ + struct adapter *sc = arg; + if_softc_ctx_t scctx = sc->shared; + uint16_t rxqid = iru->iru_qsidx; + struct em_rx_queue *que = &sc->rx_queues[rxqid]; + struct rx_ring *rxr = &que->rxr; + union e1000_rx_desc_extended *rxd; + uint64_t *paddrs; + uint32_t next_pidx, pidx; + uint16_t count; + int i; + + paddrs = iru->iru_paddrs; + pidx = iru->iru_pidx; + count = iru->iru_count; + + for (i = 0, next_pidx = pidx; i < count; i++) { + rxd = &rxr->rx_base[next_pidx]; + rxd->read.buffer_addr = htole64(paddrs[i]); + /* DD bits must be cleared */ + rxd->wb.upper.status_error = 0; + + if (++next_pidx == scctx->isc_nrxd[0]) + next_pidx = 0; + } +} + +static void +em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx) +{ + struct adapter *sc = arg; + struct em_rx_queue *que = &sc->rx_queues[rxqid]; + struct rx_ring *rxr = &que->rxr; + + E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx); +} + +static int +lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) +{ + struct adapter *sc = arg; + if_softc_ctx_t scctx = sc->shared; + struct em_rx_queue *que = &sc->rx_queues[rxqid]; + struct rx_ring *rxr = &que->rxr; + struct e1000_rx_desc *rxd; + u32 staterr = 0; + int cnt, i; + + if (budget == 1) { + rxd = (struct e1000_rx_desc *)&rxr->rx_base[idx]; + staterr = rxd->status; + return (staterr & E1000_RXD_STAT_DD); + } + + for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { + rxd = (struct e1000_rx_desc *)&rxr->rx_base[i]; + staterr = rxd->status; + + if ((staterr & E1000_RXD_STAT_DD) == 0) + break; + + if (++i == scctx->isc_nrxd[0]) + i = 0; + + if (staterr & E1000_RXD_STAT_EOP) + cnt++; + } + return (cnt); +} + +static int +em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) +{ + struct adapter *sc = arg; + if_softc_ctx_t scctx = sc->shared; + struct em_rx_queue *que = &sc->rx_queues[rxqid]; + struct rx_ring *rxr = &que->rxr; + union e1000_rx_desc_extended *rxd; + u32 staterr = 0; + int cnt, i; + + if (budget == 1) { + rxd = &rxr->rx_base[idx]; + staterr = le32toh(rxd->wb.upper.status_error); + return (staterr & E1000_RXD_STAT_DD); + } + + for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { + rxd = &rxr->rx_base[i]; + staterr = le32toh(rxd->wb.upper.status_error); + + if ((staterr & E1000_RXD_STAT_DD) == 0) + break; + + if (++i == scctx->isc_nrxd[0]) { + i = 0; + } + + if (staterr & E1000_RXD_STAT_EOP) + cnt++; + + } + return (cnt); +} + +static int +lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) +{ + struct adapter *adapter = arg; + if_softc_ctx_t scctx = adapter->shared; + struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; + struct rx_ring *rxr = &que->rxr; + struct e1000_rx_desc *rxd; + u16 len; + u32 status, errors; + bool eop; + int i, cidx; + + status = errors = i = 0; + cidx = ri->iri_cidx; + + do { + rxd = (struct e1000_rx_desc *)&rxr->rx_base[cidx]; + status = rxd->status; + errors = rxd->errors; + + /* Error Checking then decrement count */ + MPASS ((status & E1000_RXD_STAT_DD) != 0); + + len = le16toh(rxd->length); + ri->iri_len += len; + + eop = (status & E1000_RXD_STAT_EOP) != 0; + + /* Make sure bad packets are discarded */ + if (errors & E1000_RXD_ERR_FRAME_ERR_MASK) { + adapter->dropped_pkts++; + /* XXX fixup if common */ + return (EBADMSG); + } + + ri->iri_frags[i].irf_flid = 0; + ri->iri_frags[i].irf_idx = cidx; + ri->iri_frags[i].irf_len = len; + /* Zero out the receive descriptors status. */ + rxd->status = 0; + + if (++cidx == scctx->isc_nrxd[0]) + cidx = 0; + i++; + } while (!eop); + + /* XXX add a faster way to look this up */ + if (adapter->hw.mac.type >= e1000_82543 && !(status & E1000_RXD_STAT_IXSM)) + lem_receive_checksum(status, errors, ri); + + if (status & E1000_RXD_STAT_VP) { + ri->iri_vtag = le16toh(rxd->special); + ri->iri_flags |= M_VLANTAG; + } + + ri->iri_nfrags = i; + + return (0); +} + +static int +em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) +{ + struct adapter *adapter = arg; + if_softc_ctx_t scctx = adapter->shared; + struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; + struct rx_ring *rxr = &que->rxr; + union e1000_rx_desc_extended *rxd; + + u16 len; + u32 pkt_info; + u32 staterr = 0; + bool eop; + int i, cidx, vtag; + + i = vtag = 0; + cidx = ri->iri_cidx; + + do { + rxd = &rxr->rx_base[cidx]; + staterr = le32toh(rxd->wb.upper.status_error); + pkt_info = le32toh(rxd->wb.lower.mrq); + + /* Error Checking then decrement count */ + MPASS ((staterr & E1000_RXD_STAT_DD) != 0); + + len = le16toh(rxd->wb.upper.length); + ri->iri_len += len; + + eop = (staterr & E1000_RXD_STAT_EOP) != 0; + + /* Make sure bad packets are discarded */ + if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) { + adapter->dropped_pkts++; + return EBADMSG; + } + + ri->iri_frags[i].irf_flid = 0; + ri->iri_frags[i].irf_idx = cidx; + ri->iri_frags[i].irf_len = len; + /* Zero out the receive descriptors status. */ + rxd->wb.upper.status_error &= htole32(~0xFF); + + if (++cidx == scctx->isc_nrxd[0]) + cidx = 0; + i++; + } while (!eop); + + /* XXX add a faster way to look this up */ + if (adapter->hw.mac.type >= e1000_82543) + em_receive_checksum(staterr, ri); + + if (staterr & E1000_RXD_STAT_VP) { + vtag = le16toh(rxd->wb.upper.vlan); + } + + ri->iri_vtag = vtag; + if (vtag) + ri->iri_flags |= M_VLANTAG; + + ri->iri_flowid = le32toh(rxd->wb.lower.hi_dword.rss); + ri->iri_rsstype = em_determine_rsstype(pkt_info); + + ri->iri_nfrags = i; + return (0); +} + +/********************************************************************* + * + * Verify that the hardware indicated that the checksum is valid. + * Inform the stack about the status of checksum so that stack + * doesn't spend time verifying the checksum. + * + *********************************************************************/ +static void +lem_receive_checksum(int status, int errors, if_rxd_info_t ri) +{ + /* Did it pass? */ + if (status & E1000_RXD_STAT_IPCS && !(errors & E1000_RXD_ERR_IPE)) + ri->iri_csum_flags = (CSUM_IP_CHECKED|CSUM_IP_VALID); + + if (status & E1000_RXD_STAT_TCPCS) { + /* Did it pass? */ + if (!(errors & E1000_RXD_ERR_TCPE)) { + ri->iri_csum_flags |= + (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); + ri->iri_csum_data = htons(0xffff); + } + } +} + +/******************************************************************** + * + * Parse the packet type to determine the appropriate hash + * + ******************************************************************/ +static int +em_determine_rsstype(u32 pkt_info) +{ + switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) { + case E1000_RXDADV_RSSTYPE_IPV4_TCP: + return M_HASHTYPE_RSS_TCP_IPV4; + case E1000_RXDADV_RSSTYPE_IPV4: + return M_HASHTYPE_RSS_IPV4; + case E1000_RXDADV_RSSTYPE_IPV6_TCP: + return M_HASHTYPE_RSS_TCP_IPV6; + case E1000_RXDADV_RSSTYPE_IPV6_EX: + return M_HASHTYPE_RSS_IPV6_EX; + case E1000_RXDADV_RSSTYPE_IPV6: + return M_HASHTYPE_RSS_IPV6; + case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX: + return M_HASHTYPE_RSS_TCP_IPV6_EX; + default: + return M_HASHTYPE_OPAQUE; + } +} + +static void +em_receive_checksum(uint32_t status, if_rxd_info_t ri) +{ + ri->iri_csum_flags = 0; + + /* Ignore Checksum bit is set */ + if (status & E1000_RXD_STAT_IXSM) + return; + + /* If the IP checksum exists and there is no IP Checksum error */ + if ((status & (E1000_RXD_STAT_IPCS | E1000_RXDEXT_STATERR_IPE)) == + E1000_RXD_STAT_IPCS) { + ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID); + } + + /* TCP or UDP checksum */ + if ((status & (E1000_RXD_STAT_TCPCS | E1000_RXDEXT_STATERR_TCPE)) == + E1000_RXD_STAT_TCPCS) { + ri->iri_csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); + ri->iri_csum_data = htons(0xffff); + } + if (status & E1000_RXD_STAT_UDPCS) { + ri->iri_csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); + ri->iri_csum_data = htons(0xffff); + } +} diff --git a/freebsd/sys/dev/e1000/if_em.c b/freebsd/sys/dev/e1000/if_em.c index d8c1e5a6..2054f994 100644 --- a/freebsd/sys/dev/e1000/if_em.c +++ b/freebsd/sys/dev/e1000/if_em.c @@ -1,101 +1,38 @@ #include <machine/rtems-bsd-kernel-space.h> -/****************************************************************************** - - Copyright (c) 2001-2015, Intel Corporation - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - -******************************************************************************/ -/*$FreeBSD$*/ - -#include <rtems/bsd/local/opt_em.h> -#include <rtems/bsd/local/opt_ddb.h> -#include <rtems/bsd/local/opt_inet.h> -#include <rtems/bsd/local/opt_inet6.h> - -#ifdef HAVE_KERNEL_OPTION_HEADERS -#include <rtems/bsd/local/opt_device_polling.h> -#endif +/*- + * Copyright (c) 2016 Matt Macy <mmacy@nextbsd.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ -#include <rtems/bsd/sys/param.h> -#include <sys/systm.h> -#ifdef DDB -#include <sys/types.h> -#include <ddb/ddb.h> -#endif -#if __FreeBSD_version >= 800000 -#include <sys/buf_ring.h> -#endif -#include <sys/bus.h> -#include <sys/endian.h> -#include <sys/kernel.h> -#include <sys/kthread.h> -#include <sys/malloc.h> -#include <sys/mbuf.h> -#include <sys/module.h> -#include <sys/rman.h> -#include <sys/smp.h> -#include <sys/socket.h> -#include <sys/sockio.h> -#include <sys/sysctl.h> -#include <sys/taskqueue.h> -#include <sys/eventhandler.h> -#include <machine/bus.h> -#include <machine/resource.h> - -#include <net/bpf.h> -#include <net/ethernet.h> -#include <net/if.h> -#include <net/if_var.h> -#include <net/if_arp.h> -#include <net/if_dl.h> -#include <net/if_media.h> - -#include <net/if_types.h> -#include <net/if_vlan_var.h> - -#include <netinet/in_systm.h> -#include <netinet/in.h> -#include <netinet/if_ether.h> -#include <netinet/ip.h> -#include <netinet/ip6.h> -#include <netinet/tcp.h> -#include <netinet/udp.h> - -#include <machine/in_cksum.h> -#include <dev/led/led.h> -#include <dev/pci/pcivar.h> -#include <dev/pci/pcireg.h> - -#include "e1000_api.h" -#include "e1000_82571.h" +/* $FreeBSD$ */ #include "if_em.h" +#include <sys/sbuf.h> +#include <machine/_inttypes.h> + +#define em_mac_min e1000_82547 +#define igb_mac_min e1000_82575 /********************************************************************* * Driver version: @@ -112,187 +49,224 @@ char em_driver_version[] = "7.6.1-k"; * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } *********************************************************************/ -static em_vendor_info_t em_vendor_info_array[] = +static pci_vendor_info_t em_vendor_info_array[] = { - /* Intel(R) PRO/1000 Network Connection */ - { 0x8086, E1000_DEV_ID_82571EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82571EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82571EB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82572EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82572EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82572EI_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82572EI, PCI_ANY_ID, PCI_ANY_ID, 0}, - - { 0x8086, E1000_DEV_ID_82573E, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82573E_IAMT, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82573L, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82583V, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH8_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH8_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH8_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH8_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH8_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH8_82567V_3, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_BM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82574L, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82574LA, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH10_R_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH10_D_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_M_HV_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_M_HV_LC, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_D_HV_DM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_D_HV_DC, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH2_LV_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH2_LV_V, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_LPT_I217_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_LPT_I217_V, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_LPTLP_I218_LM, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_LPTLP_I218_V, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_I218_LM2, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_I218_V2, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_I218_LM3, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_I218_V3, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM2, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V2, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_LBG_I219_LM3, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM4, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V4, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM5, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V5, PCI_ANY_ID, PCI_ANY_ID, 0}, + /* Intel(R) PRO/1000 Network Connection - Legacy em*/ + PVID(0x8086, E1000_DEV_ID_82540EM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82540EM_LOM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82540EP, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82540EP_LOM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82540EP_LP, "Intel(R) PRO/1000 Network Connection"), + + PVID(0x8086, E1000_DEV_ID_82541EI, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82541ER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82541ER_LOM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82541EI_MOBILE, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82541GI, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82541GI_LF, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82541GI_MOBILE, "Intel(R) PRO/1000 Network Connection"), + + PVID(0x8086, E1000_DEV_ID_82542, "Intel(R) PRO/1000 Network Connection"), + + PVID(0x8086, E1000_DEV_ID_82543GC_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82543GC_COPPER, "Intel(R) PRO/1000 Network Connection"), + + PVID(0x8086, E1000_DEV_ID_82544EI_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82544EI_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82544GC_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82544GC_LOM, "Intel(R) PRO/1000 Network Connection"), + + PVID(0x8086, E1000_DEV_ID_82545EM_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82545EM_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82545GM_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82545GM_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82545GM_SERDES, "Intel(R) PRO/1000 Network Connection"), + + PVID(0x8086, E1000_DEV_ID_82546EB_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82546EB_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82546GB_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82546GB_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82546GB_SERDES, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82546GB_PCIE, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3, "Intel(R) PRO/1000 Network Connection"), + + PVID(0x8086, E1000_DEV_ID_82547EI, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82547EI_MOBILE, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82547GI, "Intel(R) PRO/1000 Network Connection"), + + /* Intel(R) PRO/1000 Network Connection - em */ + PVID(0x8086, E1000_DEV_ID_82571EB_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82571EB_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82571EB_SERDES, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82572EI, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82572EI_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82572EI_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82572EI_SERDES, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82573E, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82573E_IAMT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82573L, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82583V, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH8_IGP_AMT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH8_IGP_C, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH8_IFE, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH8_IFE_GT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH8_IFE_G, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH8_IGP_M, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH8_82567V_3, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_IGP_AMT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_IGP_C, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M_V, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_IFE, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_IFE_GT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_IFE_G, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_BM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82574L, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82574LA, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_LM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_LF, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_V, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_LM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_LF, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_V, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_M_HV_LM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_M_HV_LC, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_D_HV_DM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_D_HV_DC, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH2_LV_LM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH2_LV_V, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_LPT_I217_LM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_LPT_I217_V, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_LPTLP_I218_LM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_LPTLP_I218_V, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_I218_LM2, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_I218_V2, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_I218_LM3, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_I218_V3, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM2, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V2, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_LBG_I219_LM3, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM4, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V4, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM5, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V5, "Intel(R) PRO/1000 Network Connection"), /* required last entry */ - { 0, 0, 0, 0, 0} + PVID_END }; -/********************************************************************* - * Table of branding strings for all supported NICs. - *********************************************************************/ - -static char *em_strings[] = { - "Intel(R) PRO/1000 Network Connection" +static pci_vendor_info_t igb_vendor_info_array[] = +{ + /* Intel(R) PRO/1000 Network Connection - igb */ + PVID(0x8086, E1000_DEV_ID_82575EB_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576_NS, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576_NS_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576_SERDES_QUAD, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576_QUAD_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576_QUAD_COPPER_ET2, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576_VF, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82580_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82580_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82580_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82580_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82580_COPPER_DUAL, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82580_QUAD_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_DH89XXCC_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_DH89XXCC_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_DH89XXCC_SFP, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_DH89XXCC_BACKPLANE, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I350_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I350_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I350_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I350_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I350_VF, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I210_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I210_COPPER_IT, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I210_COPPER_OEM1, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I210_COPPER_FLASHLESS, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I210_SERDES_FLASHLESS, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I210_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I210_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I210_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I211_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I354_BACKPLANE_1GBPS, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I354_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"), + /* required last entry */ + PVID_END }; /********************************************************************* * Function prototypes *********************************************************************/ -static int em_probe(device_t); -static int em_attach(device_t); -static int em_detach(device_t); -static int em_shutdown(device_t); -static int em_suspend(device_t); -static int em_resume(device_t); -#ifdef EM_MULTIQUEUE -static int em_mq_start(if_t, struct mbuf *); -static int em_mq_start_locked(if_t, - struct tx_ring *); -static void em_qflush(if_t); -#else -static void em_start(if_t); -static void em_start_locked(if_t, struct tx_ring *); -#endif -static int em_ioctl(if_t, u_long, caddr_t); -static uint64_t em_get_counter(if_t, ift_counter); -static void em_init(void *); -static void em_init_locked(struct adapter *); -static void em_stop(void *); -static void em_media_status(if_t, struct ifmediareq *); -static int em_media_change(if_t); -static void em_identify_hardware(struct adapter *); -static int em_allocate_pci_resources(struct adapter *); -static int em_allocate_legacy(struct adapter *); -static int em_allocate_msix(struct adapter *); -static int em_allocate_queues(struct adapter *); -static int em_setup_msix(struct adapter *); -static void em_free_pci_resources(struct adapter *); -static void em_local_timer(void *); -static void em_reset(struct adapter *); -static int em_setup_interface(device_t, struct adapter *); -static void em_flush_desc_rings(struct adapter *); - -static void em_setup_transmit_structures(struct adapter *); -static void em_initialize_transmit_unit(struct adapter *); -static int em_allocate_transmit_buffers(struct tx_ring *); -static void em_free_transmit_structures(struct adapter *); -static void em_free_transmit_buffers(struct tx_ring *); - -static int em_setup_receive_structures(struct adapter *); -static int em_allocate_receive_buffers(struct rx_ring *); -static void em_initialize_receive_unit(struct adapter *); -static void em_free_receive_structures(struct adapter *); -static void em_free_receive_buffers(struct rx_ring *); - -static void em_enable_intr(struct adapter *); -static void em_disable_intr(struct adapter *); +static void *em_register(device_t dev); +static void *igb_register(device_t dev); +static int em_if_attach_pre(if_ctx_t ctx); +static int em_if_attach_post(if_ctx_t ctx); +static int em_if_detach(if_ctx_t ctx); +static int em_if_shutdown(if_ctx_t ctx); +static int em_if_suspend(if_ctx_t ctx); +static int em_if_resume(if_ctx_t ctx); + +static int em_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets); +static int em_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nrxqs, int nrxqsets); +static void em_if_queues_free(if_ctx_t ctx); + +static uint64_t em_if_get_counter(if_ctx_t, ift_counter); +static void em_if_init(if_ctx_t ctx); +static void em_if_stop(if_ctx_t ctx); +static void em_if_media_status(if_ctx_t, struct ifmediareq *); +static int em_if_media_change(if_ctx_t ctx); +static int em_if_mtu_set(if_ctx_t ctx, uint32_t mtu); +static void em_if_timer(if_ctx_t ctx, uint16_t qid); +static void em_if_vlan_register(if_ctx_t ctx, u16 vtag); +static void em_if_vlan_unregister(if_ctx_t ctx, u16 vtag); + +static void em_identify_hardware(if_ctx_t ctx); +static int em_allocate_pci_resources(if_ctx_t ctx); +static void em_free_pci_resources(if_ctx_t ctx); +static void em_reset(if_ctx_t ctx); +static int em_setup_interface(if_ctx_t ctx); +static int em_setup_msix(if_ctx_t ctx); + +static void em_initialize_transmit_unit(if_ctx_t ctx); +static void em_initialize_receive_unit(if_ctx_t ctx); + +static void em_if_enable_intr(if_ctx_t ctx); +static void em_if_disable_intr(if_ctx_t ctx); +static int em_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid); +static int em_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid); +static void em_if_multi_set(if_ctx_t ctx); +static void em_if_update_admin_status(if_ctx_t ctx); +static void em_if_debug(if_ctx_t ctx); static void em_update_stats_counters(struct adapter *); static void em_add_hw_stats(struct adapter *adapter); -static void em_txeof(struct tx_ring *); -static bool em_rxeof(struct rx_ring *, int, int *); -#ifndef __NO_STRICT_ALIGNMENT -static int em_fixup_rx(struct rx_ring *); -#endif -static void em_setup_rxdesc(union e1000_rx_desc_extended *, - const struct em_rxbuffer *rxbuf); -static void em_receive_checksum(uint32_t status, struct mbuf *); -static void em_transmit_checksum_setup(struct tx_ring *, struct mbuf *, int, - struct ip *, u32 *, u32 *); -static void em_tso_setup(struct tx_ring *, struct mbuf *, int, struct ip *, - struct tcphdr *, u32 *, u32 *); -static void em_set_promisc(struct adapter *); -static void em_disable_promisc(struct adapter *); -static void em_set_multi(struct adapter *); -static void em_update_link_status(struct adapter *); -static void em_refresh_mbufs(struct rx_ring *, int); -static void em_register_vlan(void *, if_t, u16); -static void em_unregister_vlan(void *, if_t, u16); +static int em_if_set_promisc(if_ctx_t ctx, int flags); static void em_setup_vlan_hw_support(struct adapter *); -static int em_xmit(struct tx_ring *, struct mbuf **); -static int em_dma_malloc(struct adapter *, bus_size_t, - struct em_dma_alloc *, int); -static void em_dma_free(struct adapter *, struct em_dma_alloc *); static int em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS); static void em_print_nvm_info(struct adapter *); static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS); +static int em_get_rs(SYSCTL_HANDLER_ARGS); static void em_print_debug_info(struct adapter *); static int em_is_valid_ether_addr(u8 *); static int em_sysctl_int_delay(SYSCTL_HANDLER_ARGS); @@ -301,65 +275,123 @@ static void em_add_int_delay_sysctl(struct adapter *, const char *, /* Management and WOL Support */ static void em_init_manageability(struct adapter *); static void em_release_manageability(struct adapter *); -static void em_get_hw_control(struct adapter *); -static void em_release_hw_control(struct adapter *); -static void em_get_wakeup(device_t); -static void em_enable_wakeup(device_t); +static void em_get_hw_control(struct adapter *); +static void em_release_hw_control(struct adapter *); +static void em_get_wakeup(if_ctx_t ctx); +static void em_enable_wakeup(if_ctx_t ctx); static int em_enable_phy_wakeup(struct adapter *); -static void em_led_func(void *, int); static void em_disable_aspm(struct adapter *); -static int em_irq_fast(void *); +int em_intr(void *arg); +static void em_disable_promisc(if_ctx_t ctx); /* MSIX handlers */ -static void em_msix_tx(void *); -static void em_msix_rx(void *); -static void em_msix_link(void *); -static void em_handle_tx(void *context, int pending); -static void em_handle_rx(void *context, int pending); -static void em_handle_link(void *context, int pending); - -#ifdef EM_MULTIQUEUE -static void em_enable_vectors_82574(struct adapter *); -#endif +static int em_if_msix_intr_assign(if_ctx_t, int); +static int em_msix_link(void *); +static void em_handle_link(void *context); + +static void em_enable_vectors_82574(if_ctx_t); -static void em_set_sysctl_value(struct adapter *, const char *, - const char *, int *, int); static int em_set_flowcntl(SYSCTL_HANDLER_ARGS); static int em_sysctl_eee(SYSCTL_HANDLER_ARGS); +static void em_if_led_func(if_ctx_t ctx, int onoff); -static __inline void em_rx_discard(struct rx_ring *, int); +static int em_get_regs(SYSCTL_HANDLER_ARGS); + +static void lem_smartspeed(struct adapter *adapter); +static void igb_configure_queues(struct adapter *adapter); -#ifdef DEVICE_POLLING -static poll_handler_t em_poll; -#endif /* POLLING */ /********************************************************************* * FreeBSD Device Interface Entry Points *********************************************************************/ - static device_method_t em_methods[] = { /* Device interface */ - DEVMETHOD(device_probe, em_probe), - DEVMETHOD(device_attach, em_attach), - DEVMETHOD(device_detach, em_detach), - DEVMETHOD(device_shutdown, em_shutdown), - DEVMETHOD(device_suspend, em_suspend), - DEVMETHOD(device_resume, em_resume), + DEVMETHOD(device_register, em_register), + DEVMETHOD(device_probe, iflib_device_probe), + DEVMETHOD(device_attach, iflib_device_attach), + DEVMETHOD(device_detach, iflib_device_detach), + DEVMETHOD(device_shutdown, iflib_device_shutdown), + DEVMETHOD(device_suspend, iflib_device_suspend), + DEVMETHOD(device_resume, iflib_device_resume), + DEVMETHOD_END +}; + +static device_method_t igb_methods[] = { + /* Device interface */ + DEVMETHOD(device_register, igb_register), + DEVMETHOD(device_probe, iflib_device_probe), + DEVMETHOD(device_attach, iflib_device_attach), + DEVMETHOD(device_detach, iflib_device_detach), + DEVMETHOD(device_shutdown, iflib_device_shutdown), + DEVMETHOD(device_suspend, iflib_device_suspend), + DEVMETHOD(device_resume, iflib_device_resume), DEVMETHOD_END }; + static driver_t em_driver = { "em", em_methods, sizeof(struct adapter), }; -devclass_t em_devclass; +static devclass_t em_devclass; DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0); + MODULE_DEPEND(em, pci, 1, 1, 1); MODULE_DEPEND(em, ether, 1, 1, 1); -#ifdef DEV_NETMAP -MODULE_DEPEND(em, netmap, 1, 1, 1); -#endif /* DEV_NETMAP */ +MODULE_DEPEND(em, iflib, 1, 1, 1); + +static driver_t igb_driver = { + "igb", igb_methods, sizeof(struct adapter), +}; + +static devclass_t igb_devclass; +DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0); + +MODULE_DEPEND(igb, pci, 1, 1, 1); +MODULE_DEPEND(igb, ether, 1, 1, 1); +MODULE_DEPEND(igb, iflib, 1, 1, 1); + + +static device_method_t em_if_methods[] = { + DEVMETHOD(ifdi_attach_pre, em_if_attach_pre), + DEVMETHOD(ifdi_attach_post, em_if_attach_post), + DEVMETHOD(ifdi_detach, em_if_detach), + DEVMETHOD(ifdi_shutdown, em_if_shutdown), + DEVMETHOD(ifdi_suspend, em_if_suspend), + DEVMETHOD(ifdi_resume, em_if_resume), + DEVMETHOD(ifdi_init, em_if_init), + DEVMETHOD(ifdi_stop, em_if_stop), + DEVMETHOD(ifdi_msix_intr_assign, em_if_msix_intr_assign), + DEVMETHOD(ifdi_intr_enable, em_if_enable_intr), + DEVMETHOD(ifdi_intr_disable, em_if_disable_intr), + DEVMETHOD(ifdi_tx_queues_alloc, em_if_tx_queues_alloc), + DEVMETHOD(ifdi_rx_queues_alloc, em_if_rx_queues_alloc), + DEVMETHOD(ifdi_queues_free, em_if_queues_free), + DEVMETHOD(ifdi_update_admin_status, em_if_update_admin_status), + DEVMETHOD(ifdi_multi_set, em_if_multi_set), + DEVMETHOD(ifdi_media_status, em_if_media_status), + DEVMETHOD(ifdi_media_change, em_if_media_change), + DEVMETHOD(ifdi_mtu_set, em_if_mtu_set), + DEVMETHOD(ifdi_promisc_set, em_if_set_promisc), + DEVMETHOD(ifdi_timer, em_if_timer), + DEVMETHOD(ifdi_vlan_register, em_if_vlan_register), + DEVMETHOD(ifdi_vlan_unregister, em_if_vlan_unregister), + DEVMETHOD(ifdi_get_counter, em_if_get_counter), + DEVMETHOD(ifdi_led_func, em_if_led_func), + DEVMETHOD(ifdi_rx_queue_intr_enable, em_if_rx_queue_intr_enable), + DEVMETHOD(ifdi_tx_queue_intr_enable, em_if_tx_queue_intr_enable), + DEVMETHOD(ifdi_debug, em_if_debug), + DEVMETHOD_END +}; + +/* + * note that if (adapter->msix_mem) is replaced by: + * if (adapter->intr_type == IFLIB_INTR_MSIX) + */ +static driver_t em_if_driver = { + "em_if", em_if_methods, sizeof(struct adapter) +}; /********************************************************************* * Tunable default values. @@ -367,10 +399,16 @@ MODULE_DEPEND(em, netmap, 1, 1, 1); #define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000) #define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024) +#define M_TSO_LEN 66 #define MAX_INTS_PER_SEC 8000 #define DEFAULT_ITR (1000000000/(MAX_INTS_PER_SEC * 256)) +/* Allow common code without TSO */ +#ifndef CSUM_TSO +#define CSUM_TSO 0 +#endif + #define TSO_WORKAROUND 4 static SYSCTL_NODE(_hw, OID_AUTO, em, CTLFLAG_RD, 0, "EM driver parameters"); @@ -395,39 +433,15 @@ SYSCTL_INT(_hw_em, OID_AUTO, rx_abs_int_delay, CTLFLAG_RDTUN, &em_rx_abs_int_delay_dflt, 0, "Default receive interrupt delay limit in usecs"); -static int em_rxd = EM_DEFAULT_RXD; -static int em_txd = EM_DEFAULT_TXD; -SYSCTL_INT(_hw_em, OID_AUTO, rxd, CTLFLAG_RDTUN, &em_rxd, 0, - "Number of receive descriptors per queue"); -SYSCTL_INT(_hw_em, OID_AUTO, txd, CTLFLAG_RDTUN, &em_txd, 0, - "Number of transmit descriptors per queue"); - static int em_smart_pwr_down = FALSE; SYSCTL_INT(_hw_em, OID_AUTO, smart_pwr_down, CTLFLAG_RDTUN, &em_smart_pwr_down, 0, "Set to true to leave smart power down enabled on newer adapters"); /* Controls whether promiscuous also shows bad packets */ -static int em_debug_sbp = FALSE; +static int em_debug_sbp = TRUE; SYSCTL_INT(_hw_em, OID_AUTO, sbp, CTLFLAG_RDTUN, &em_debug_sbp, 0, "Show bad packets in promiscuous mode"); -static int em_enable_msix = TRUE; -SYSCTL_INT(_hw_em, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &em_enable_msix, 0, - "Enable MSI-X interrupts"); - -#ifdef EM_MULTIQUEUE -static int em_num_queues = 1; -SYSCTL_INT(_hw_em, OID_AUTO, num_queues, CTLFLAG_RDTUN, &em_num_queues, 0, - "82574 only: Number of queues to configure, 0 indicates autoconfigure"); -#endif - -/* -** Global variable to store last used CPU when binding queues -** to CPUs in igb_allocate_msix. Starts at CPU_FIRST and increments when a -** queue is bound to a cpu. -*/ -static int em_last_bind_cpu = -1; - /* How many packets rxeof tries to clean at a time */ static int em_rx_process_limit = 100; SYSCTL_INT(_hw_em, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, @@ -440,64 +454,243 @@ static int eee_setting = 1; SYSCTL_INT(_hw_em, OID_AUTO, eee_setting, CTLFLAG_RDTUN, &eee_setting, 0, "Enable Energy Efficient Ethernet"); +/* +** Tuneable Interrupt rate +*/ +static int em_max_interrupt_rate = 8000; +SYSCTL_INT(_hw_em, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN, + &em_max_interrupt_rate, 0, "Maximum interrupts per second"); + + + /* Global used in WOL setup with multiport cards */ static int global_quad_port_a = 0; -#ifdef DEV_NETMAP /* see ixgbe.c for details */ -#include <dev/netmap/if_em_netmap.h> -#endif /* DEV_NETMAP */ +extern struct if_txrx igb_txrx; +extern struct if_txrx em_txrx; +extern struct if_txrx lem_txrx; + +static struct if_shared_ctx em_sctx_init = { + .isc_magic = IFLIB_MAGIC, + .isc_q_align = PAGE_SIZE, + .isc_tx_maxsize = EM_TSO_SIZE, + .isc_tx_maxsegsize = PAGE_SIZE, + .isc_rx_maxsize = MJUM9BYTES, + .isc_rx_nsegments = 1, + .isc_rx_maxsegsize = MJUM9BYTES, + .isc_nfl = 1, + .isc_nrxqs = 1, + .isc_ntxqs = 1, + .isc_admin_intrcnt = 1, + .isc_vendor_info = em_vendor_info_array, + .isc_driver_version = em_driver_version, + .isc_driver = &em_if_driver, + .isc_flags = IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP, + + .isc_nrxd_min = {EM_MIN_RXD}, + .isc_ntxd_min = {EM_MIN_TXD}, + .isc_nrxd_max = {EM_MAX_RXD}, + .isc_ntxd_max = {EM_MAX_TXD}, + .isc_nrxd_default = {EM_DEFAULT_RXD}, + .isc_ntxd_default = {EM_DEFAULT_TXD}, +}; -/********************************************************************* - * Device identification routine +if_shared_ctx_t em_sctx = &em_sctx_init; + + +static struct if_shared_ctx igb_sctx_init = { + .isc_magic = IFLIB_MAGIC, + .isc_q_align = PAGE_SIZE, + .isc_tx_maxsize = EM_TSO_SIZE, + .isc_tx_maxsegsize = PAGE_SIZE, + .isc_rx_maxsize = MJUM9BYTES, + .isc_rx_nsegments = 1, + .isc_rx_maxsegsize = MJUM9BYTES, + .isc_nfl = 1, + .isc_nrxqs = 1, + .isc_ntxqs = 1, + .isc_admin_intrcnt = 1, + .isc_vendor_info = igb_vendor_info_array, + .isc_driver_version = em_driver_version, + .isc_driver = &em_if_driver, + .isc_flags = IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP, + + .isc_nrxd_min = {EM_MIN_RXD}, + .isc_ntxd_min = {EM_MIN_TXD}, + .isc_nrxd_max = {EM_MAX_RXD}, + .isc_ntxd_max = {EM_MAX_TXD}, + .isc_nrxd_default = {EM_DEFAULT_RXD}, + .isc_ntxd_default = {EM_DEFAULT_TXD}, +}; + +if_shared_ctx_t igb_sctx = &igb_sctx_init; + +/***************************************************************** * - * em_probe determines if the driver should be loaded on - * adapter based on PCI vendor/device id of the adapter. + * Dump Registers * - * return BUS_PROBE_DEFAULT on success, positive on failure - *********************************************************************/ + ****************************************************************/ +#define IGB_REGS_LEN 739 -static int -em_probe(device_t dev) +static int em_get_regs(SYSCTL_HANDLER_ARGS) { - char adapter_name[60]; - uint16_t pci_vendor_id = 0; - uint16_t pci_device_id = 0; - uint16_t pci_subvendor_id = 0; - uint16_t pci_subdevice_id = 0; - em_vendor_info_t *ent; + struct adapter *adapter = (struct adapter *)arg1; + struct e1000_hw *hw = &adapter->hw; - INIT_DEBUGOUT("em_probe: begin"); + struct sbuf *sb; + u32 *regs_buff = (u32 *)malloc(sizeof(u32) * IGB_REGS_LEN, M_DEVBUF, M_NOWAIT); + int rc; - pci_vendor_id = pci_get_vendor(dev); - if (pci_vendor_id != EM_VENDOR_ID) - return (ENXIO); + memset(regs_buff, 0, IGB_REGS_LEN * sizeof(u32)); - pci_device_id = pci_get_device(dev); - pci_subvendor_id = pci_get_subvendor(dev); - pci_subdevice_id = pci_get_subdevice(dev); - - ent = em_vendor_info_array; - while (ent->vendor_id != 0) { - if ((pci_vendor_id == ent->vendor_id) && - (pci_device_id == ent->device_id) && - - ((pci_subvendor_id == ent->subvendor_id) || - (ent->subvendor_id == PCI_ANY_ID)) && - - ((pci_subdevice_id == ent->subdevice_id) || - (ent->subdevice_id == PCI_ANY_ID))) { - sprintf(adapter_name, "%s %s", - em_strings[ent->index], - em_driver_version); - device_set_desc_copy(dev, adapter_name); - return (BUS_PROBE_DEFAULT); - } - ent++; + rc = sysctl_wire_old_buffer(req, 0); + MPASS(rc == 0); + if (rc != 0) + return (rc); + + sb = sbuf_new_for_sysctl(NULL, NULL, 32*400, req); + MPASS(sb != NULL); + if (sb == NULL) + return (ENOMEM); + + /* General Registers */ + regs_buff[0] = E1000_READ_REG(hw, E1000_CTRL); + regs_buff[1] = E1000_READ_REG(hw, E1000_STATUS); + regs_buff[2] = E1000_READ_REG(hw, E1000_CTRL_EXT); + regs_buff[3] = E1000_READ_REG(hw, E1000_ICR); + regs_buff[4] = E1000_READ_REG(hw, E1000_RCTL); + regs_buff[5] = E1000_READ_REG(hw, E1000_RDLEN(0)); + regs_buff[6] = E1000_READ_REG(hw, E1000_RDH(0)); + regs_buff[7] = E1000_READ_REG(hw, E1000_RDT(0)); + regs_buff[8] = E1000_READ_REG(hw, E1000_RXDCTL(0)); + regs_buff[9] = E1000_READ_REG(hw, E1000_RDBAL(0)); + regs_buff[10] = E1000_READ_REG(hw, E1000_RDBAH(0)); + regs_buff[11] = E1000_READ_REG(hw, E1000_TCTL); + regs_buff[12] = E1000_READ_REG(hw, E1000_TDBAL(0)); + regs_buff[13] = E1000_READ_REG(hw, E1000_TDBAH(0)); + regs_buff[14] = E1000_READ_REG(hw, E1000_TDLEN(0)); + regs_buff[15] = E1000_READ_REG(hw, E1000_TDH(0)); + regs_buff[16] = E1000_READ_REG(hw, E1000_TDT(0)); + regs_buff[17] = E1000_READ_REG(hw, E1000_TXDCTL(0)); + regs_buff[18] = E1000_READ_REG(hw, E1000_TDFH); + regs_buff[19] = E1000_READ_REG(hw, E1000_TDFT); + regs_buff[20] = E1000_READ_REG(hw, E1000_TDFHS); + regs_buff[21] = E1000_READ_REG(hw, E1000_TDFPC); + + sbuf_printf(sb, "General Registers\n"); + sbuf_printf(sb, "\tCTRL\t %08x\n", regs_buff[0]); + sbuf_printf(sb, "\tSTATUS\t %08x\n", regs_buff[1]); + sbuf_printf(sb, "\tCTRL_EXIT\t %08x\n\n", regs_buff[2]); + + sbuf_printf(sb, "Interrupt Registers\n"); + sbuf_printf(sb, "\tICR\t %08x\n\n", regs_buff[3]); + + sbuf_printf(sb, "RX Registers\n"); + sbuf_printf(sb, "\tRCTL\t %08x\n", regs_buff[4]); + sbuf_printf(sb, "\tRDLEN\t %08x\n", regs_buff[5]); + sbuf_printf(sb, "\tRDH\t %08x\n", regs_buff[6]); + sbuf_printf(sb, "\tRDT\t %08x\n", regs_buff[7]); + sbuf_printf(sb, "\tRXDCTL\t %08x\n", regs_buff[8]); + sbuf_printf(sb, "\tRDBAL\t %08x\n", regs_buff[9]); + sbuf_printf(sb, "\tRDBAH\t %08x\n\n", regs_buff[10]); + + sbuf_printf(sb, "TX Registers\n"); + sbuf_printf(sb, "\tTCTL\t %08x\n", regs_buff[11]); + sbuf_printf(sb, "\tTDBAL\t %08x\n", regs_buff[12]); + sbuf_printf(sb, "\tTDBAH\t %08x\n", regs_buff[13]); + sbuf_printf(sb, "\tTDLEN\t %08x\n", regs_buff[14]); + sbuf_printf(sb, "\tTDH\t %08x\n", regs_buff[15]); + sbuf_printf(sb, "\tTDT\t %08x\n", regs_buff[16]); + sbuf_printf(sb, "\tTXDCTL\t %08x\n", regs_buff[17]); + sbuf_printf(sb, "\tTDFH\t %08x\n", regs_buff[18]); + sbuf_printf(sb, "\tTDFT\t %08x\n", regs_buff[19]); + sbuf_printf(sb, "\tTDFHS\t %08x\n", regs_buff[20]); + sbuf_printf(sb, "\tTDFPC\t %08x\n\n", regs_buff[21]); + +#ifdef DUMP_DESCS + { + if_softc_ctx_t scctx = adapter->shared; + struct rx_ring *rxr = &rx_que->rxr; + struct tx_ring *txr = &tx_que->txr; + int ntxd = scctx->isc_ntxd[0]; + int nrxd = scctx->isc_nrxd[0]; + int j; + + for (j = 0; j < nrxd; j++) { + u32 staterr = le32toh(rxr->rx_base[j].wb.upper.status_error); + u32 length = le32toh(rxr->rx_base[j].wb.upper.length); + sbuf_printf(sb, "\tReceive Descriptor Address %d: %08" PRIx64 " Error:%d Length:%d\n", j, rxr->rx_base[j].read.buffer_addr, staterr, length); + } + + for (j = 0; j < min(ntxd, 256); j++) { + unsigned int *ptr = (unsigned int *)&txr->tx_base[j]; + + sbuf_printf(sb, "\tTXD[%03d] [0]: %08x [1]: %08x [2]: %08x [3]: %08x eop: %d DD=%d\n", + j, ptr[0], ptr[1], ptr[2], ptr[3], buf->eop, + buf->eop != -1 ? txr->tx_base[buf->eop].upper.fields.status & E1000_TXD_STAT_DD : 0); + + } } +#endif - return (ENXIO); + rc = sbuf_finish(sb); + sbuf_delete(sb); + return(rc); } +static void * +em_register(device_t dev) +{ + return (em_sctx); +} + +static void * +igb_register(device_t dev) +{ + return (igb_sctx); +} + +static int +em_set_num_queues(if_ctx_t ctx) +{ + struct adapter *adapter = iflib_get_softc(ctx); + int maxqueues; + + /* Sanity check based on HW */ + switch (adapter->hw.mac.type) { + case e1000_82576: + case e1000_82580: + case e1000_i350: + case e1000_i354: + maxqueues = 8; + break; + case e1000_i210: + case e1000_82575: + maxqueues = 4; + break; + case e1000_i211: + case e1000_82574: + maxqueues = 2; + break; + default: + maxqueues = 1; + break; + } + + return (maxqueues); +} + + +#define EM_CAPS \ + IFCAP_TSO4 | IFCAP_TXCSUM | IFCAP_LRO | IFCAP_RXCSUM | IFCAP_VLAN_HWFILTER | IFCAP_WOL_MAGIC | \ + IFCAP_WOL_MCAST | IFCAP_WOL | IFCAP_VLAN_HWTSO | IFCAP_HWCSUM | IFCAP_VLAN_HWTAGGING | \ + IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWTSO | IFCAP_VLAN_MTU; + +#define IGB_CAPS \ + IFCAP_TSO4 | IFCAP_TXCSUM | IFCAP_LRO | IFCAP_RXCSUM | IFCAP_VLAN_HWFILTER | IFCAP_WOL_MAGIC | \ + IFCAP_WOL_MCAST | IFCAP_WOL | IFCAP_VLAN_HWTSO | IFCAP_HWCSUM | IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM | \ + IFCAP_VLAN_HWTSO | IFCAP_VLAN_MTU | IFCAP_TXCSUM_IPV6 | IFCAP_HWCSUM_IPV6 | IFCAP_JUMBO_MTU; + /********************************************************************* * Device initialization routine * @@ -509,23 +702,30 @@ em_probe(device_t dev) *********************************************************************/ static int -em_attach(device_t dev) +em_if_attach_pre(if_ctx_t ctx) { - struct adapter *adapter; - struct e1000_hw *hw; - int error = 0; + struct adapter *adapter; + if_softc_ctx_t scctx; + device_t dev; + struct e1000_hw *hw; + int error = 0; - INIT_DEBUGOUT("em_attach: begin"); + INIT_DEBUGOUT("em_if_attach_pre begin"); + dev = iflib_get_dev(ctx); + adapter = iflib_get_softc(ctx); if (resource_disabled("em", device_get_unit(dev))) { device_printf(dev, "Disabled by device hint\n"); return (ENXIO); } - adapter = device_get_softc(dev); + adapter->ctx = ctx; adapter->dev = adapter->osdep.dev = dev; + scctx = adapter->shared = iflib_get_softc_ctx(ctx); + adapter->media = iflib_get_media(ctx); hw = &adapter->hw; - EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); + + adapter->tx_process_limit = scctx->isc_ntxd[0]; /* SYSCTL stuff */ SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), @@ -543,13 +743,78 @@ em_attach(device_t dev) OID_AUTO, "fc", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, em_set_flowcntl, "I", "Flow Control"); - callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "reg_dump", CTLTYPE_STRING | CTLFLAG_RD, adapter, 0, + em_get_regs, "A", "Dump Registers"); + + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "rs_dump", CTLTYPE_INT | CTLFLAG_RW, adapter, 0, + em_get_rs, "I", "Dump RS indexes"); /* Determine hardware and mac info */ - em_identify_hardware(adapter); + em_identify_hardware(ctx); + + /* Set isc_msix_bar */ + scctx->isc_msix_bar = PCIR_BAR(EM_MSIX_BAR); + scctx->isc_tx_nsegments = EM_MAX_SCATTER; + scctx->isc_tx_tso_segments_max = scctx->isc_tx_nsegments; + scctx->isc_tx_tso_size_max = EM_TSO_SIZE; + scctx->isc_tx_tso_segsize_max = EM_TSO_SEG_SIZE; + scctx->isc_nrxqsets_max = scctx->isc_ntxqsets_max = em_set_num_queues(ctx); + device_printf(dev, "attach_pre capping queues at %d\n", scctx->isc_ntxqsets_max); + + scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO; + + + if (adapter->hw.mac.type >= igb_mac_min) { + int try_second_bar; + + scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] * sizeof(union e1000_adv_tx_desc), EM_DBA_ALIGN); + scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] * sizeof(union e1000_adv_rx_desc), EM_DBA_ALIGN); + scctx->isc_txd_size[0] = sizeof(union e1000_adv_tx_desc); + scctx->isc_rxd_size[0] = sizeof(union e1000_adv_rx_desc); + scctx->isc_txrx = &igb_txrx; + scctx->isc_capenable = IGB_CAPS; + scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_TSO | CSUM_IP6_TCP \ + | CSUM_IP6_UDP | CSUM_IP6_TCP; + if (adapter->hw.mac.type != e1000_82575) + scctx->isc_tx_csum_flags |= CSUM_SCTP | CSUM_IP6_SCTP; + + /* + ** Some new devices, as with ixgbe, now may + ** use a different BAR, so we need to keep + ** track of which is used. + */ + try_second_bar = pci_read_config(dev, scctx->isc_msix_bar, 4); + if (try_second_bar == 0) + scctx->isc_msix_bar += 4; + + } else if (adapter->hw.mac.type >= em_mac_min) { + scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0]* sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); + scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] * sizeof(union e1000_rx_desc_extended), EM_DBA_ALIGN); + scctx->isc_txd_size[0] = sizeof(struct e1000_tx_desc); + scctx->isc_rxd_size[0] = sizeof(union e1000_rx_desc_extended); + scctx->isc_txrx = &em_txrx; + scctx->isc_capenable = EM_CAPS; + scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO; + } else { + scctx->isc_txqsizes[0] = roundup2((scctx->isc_ntxd[0] + 1) * sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); + scctx->isc_rxqsizes[0] = roundup2((scctx->isc_nrxd[0] + 1) * sizeof(struct e1000_rx_desc), EM_DBA_ALIGN); + scctx->isc_txd_size[0] = sizeof(struct e1000_tx_desc); + scctx->isc_rxd_size[0] = sizeof(struct e1000_rx_desc); + scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO; + scctx->isc_txrx = &lem_txrx; + scctx->isc_capenable = EM_CAPS; + if (adapter->hw.mac.type < e1000_82543) + scctx->isc_capenable &= ~(IFCAP_HWCSUM|IFCAP_VLAN_HWCSUM); + scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO; + scctx->isc_msix_bar = 0; + } /* Setup PCI resources */ - if (em_allocate_pci_resources(adapter)) { + if (em_allocate_pci_resources(ctx)) { device_printf(dev, "Allocation of PCI resources failed\n"); error = ENXIO; goto err_pci; @@ -558,7 +823,7 @@ em_attach(device_t dev) /* ** For ICH8 and family we need to ** map the flash memory, and this - ** must happen after the MAC is + ** must happen after the MAC is ** identified */ if ((hw->mac.type == e1000_ich8lan) || @@ -605,11 +870,7 @@ em_attach(device_t dev) goto err_pci; } - /* - * Setup MSI/X or MSI if PCI Express - */ - adapter->msix = em_setup_msix(adapter); - + em_setup_msix(ctx); e1000_get_bus_info(hw); /* Set up some sysctls for the tunable interrupt delays */ @@ -635,36 +896,14 @@ em_attach(device_t dev) E1000_REGISTER(hw, E1000_ITR), DEFAULT_ITR); - /* Sysctl for limiting the amount of work done in the taskqueue */ - em_set_sysctl_value(adapter, "rx_processing_limit", - "max number of rx packets to process", &adapter->rx_process_limit, - em_rx_process_limit); - - /* - * Validate number of transmit and receive descriptors. It - * must not exceed hardware maximum, and must be multiple - * of E1000_DBA_ALIGN. - */ - if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 || - (em_txd > EM_MAX_TXD) || (em_txd < EM_MIN_TXD)) { - device_printf(dev, "Using %d TX descriptors instead of %d!\n", - EM_DEFAULT_TXD, em_txd); - adapter->num_tx_desc = EM_DEFAULT_TXD; - } else - adapter->num_tx_desc = em_txd; - - if (((em_rxd * sizeof(union e1000_rx_desc_extended)) % EM_DBA_ALIGN) != 0 || - (em_rxd > EM_MAX_RXD) || (em_rxd < EM_MIN_RXD)) { - device_printf(dev, "Using %d RX descriptors instead of %d!\n", - EM_DEFAULT_RXD, em_rxd); - adapter->num_rx_desc = EM_DEFAULT_RXD; - } else - adapter->num_rx_desc = em_rxd; - hw->mac.autoneg = DO_AUTO_NEG; hw->phy.autoneg_wait_to_complete = FALSE; hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; + if (adapter->hw.mac.type < em_mac_min) { + e1000_init_script_state_82541(&adapter->hw, TRUE); + e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE); + } /* Copper options */ if (hw->phy.media_type == e1000_media_type_copper) { hw->phy.mdix = AUTO_ALL_MODES; @@ -676,7 +915,7 @@ em_attach(device_t dev) * Set the frame limits assuming * standard ethernet sized frames. */ - adapter->hw.mac.max_frame_size = + scctx->isc_max_frame_size = adapter->hw.mac.max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; /* @@ -685,14 +924,6 @@ em_attach(device_t dev) */ hw->mac.report_tx_early = 1; - /* - ** Get queue/ring memory - */ - if (em_allocate_queues(adapter)) { - error = ENOMEM; - goto err_pci; - } - /* Allocate multicast array memory. */ adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); @@ -705,7 +936,7 @@ em_attach(device_t dev) /* Check SOL/IDER usage */ if (e1000_check_reset_block(hw)) device_printf(dev, "PHY reset is blocked" - " due to SOL/IDER session.\n"); + " due to SOL/IDER session.\n"); /* Sysctl for setting Energy Efficient Ethernet */ hw->dev_spec.ich8lan.eee_disable = eee_setting; @@ -722,7 +953,6 @@ em_attach(device_t dev) */ e1000_reset_hw(hw); - /* Make sure we have a good EEPROM before we read from it */ if (e1000_validate_nvm_checksum(hw) < 0) { /* @@ -741,7 +971,7 @@ em_attach(device_t dev) /* Copy the permanent MAC address out of the EEPROM */ if (e1000_read_mac_addr(hw) < 0) { device_printf(dev, "EEPROM read error while reading MAC" - " address\n"); + " address\n"); error = EIO; goto err_late; } @@ -756,67 +986,57 @@ em_attach(device_t dev) e1000_disable_ulp_lpt_lp(hw, TRUE); /* - ** Do interrupt configuration - */ - if (adapter->msix > 1) /* Do MSIX */ - error = em_allocate_msix(adapter); - else /* MSI or Legacy */ - error = em_allocate_legacy(adapter); - if (error) - goto err_late; - - /* * Get Wake-on-Lan and Management info for later use */ - em_get_wakeup(dev); + em_get_wakeup(ctx); + + iflib_set_mac(ctx, hw->mac.addr); + + return (0); + +err_late: + em_release_hw_control(adapter); +err_pci: + em_free_pci_resources(ctx); + free(adapter->mta, M_DEVBUF); + + return (error); +} +static int +em_if_attach_post(if_ctx_t ctx) +{ + struct adapter *adapter = iflib_get_softc(ctx); + struct e1000_hw *hw = &adapter->hw; + int error = 0; + /* Setup OS specific network interface */ - if (em_setup_interface(dev, adapter) != 0) + error = em_setup_interface(ctx); + if (error != 0) { goto err_late; + } - em_reset(adapter); + em_reset(ctx); /* Initialize statistics */ em_update_stats_counters(adapter); - hw->mac.get_link_status = 1; - em_update_link_status(adapter); - - /* Register for VLAN events */ - adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, - em_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); - adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, - em_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); - + em_if_update_admin_status(ctx); em_add_hw_stats(adapter); /* Non-AMT based hardware can now take control from firmware */ if (adapter->has_manage && !adapter->has_amt) em_get_hw_control(adapter); + + INIT_DEBUGOUT("em_if_attach_post: end"); - /* Tell the stack that the interface is not active */ - if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); - - adapter->led_dev = led_create(em_led_func, adapter, - device_get_nameunit(dev)); -#ifdef DEV_NETMAP - em_netmap_attach(adapter); -#endif /* DEV_NETMAP */ - - INIT_DEBUGOUT("em_attach: end"); - - return (0); + return (error); err_late: - em_free_transmit_structures(adapter); - em_free_receive_structures(adapter); em_release_hw_control(adapter); - if (adapter->ifp != (void *)NULL) - if_free(adapter->ifp); -err_pci: - em_free_pci_resources(adapter); + em_free_pci_resources(ctx); + em_if_queues_free(ctx); free(adapter->mta, M_DEVBUF); - EM_CORE_LOCK_DESTROY(adapter); return (error); } @@ -832,60 +1052,17 @@ err_pci: *********************************************************************/ static int -em_detach(device_t dev) +em_if_detach(if_ctx_t ctx) { - struct adapter *adapter = device_get_softc(dev); - if_t ifp = adapter->ifp; + struct adapter *adapter = iflib_get_softc(ctx); INIT_DEBUGOUT("em_detach: begin"); - /* Make sure VLANS are not using driver */ - if (if_vlantrunkinuse(ifp)) { - device_printf(dev,"Vlan in use, detach first\n"); - return (EBUSY); - } - -#ifdef DEVICE_POLLING - if (if_getcapenable(ifp) & IFCAP_POLLING) - ether_poll_deregister(ifp); -#endif - - if (adapter->led_dev != NULL) - led_destroy(adapter->led_dev); - - EM_CORE_LOCK(adapter); - adapter->in_detach = 1; - em_stop(adapter); - EM_CORE_UNLOCK(adapter); - EM_CORE_LOCK_DESTROY(adapter); - e1000_phy_hw_reset(&adapter->hw); em_release_manageability(adapter); em_release_hw_control(adapter); - - /* Unregister VLAN events */ - if (adapter->vlan_attach != NULL) - EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); - if (adapter->vlan_detach != NULL) - EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); - - ether_ifdetach(adapter->ifp); - callout_drain(&adapter->timer); - -#ifdef DEV_NETMAP - netmap_detach(ifp); -#endif /* DEV_NETMAP */ - - em_free_pci_resources(adapter); - bus_generic_detach(dev); - if_free(ifp); - - em_free_transmit_structures(adapter); - em_free_receive_structures(adapter); - - em_release_hw_control(adapter); - free(adapter->mta, M_DEVBUF); + em_free_pci_resources(ctx); return (0); } @@ -897,448 +1074,84 @@ em_detach(device_t dev) **********************************************************************/ static int -em_shutdown(device_t dev) +em_if_shutdown(if_ctx_t ctx) { - return em_suspend(dev); + return em_if_suspend(ctx); } /* * Suspend/resume device methods. */ static int -em_suspend(device_t dev) +em_if_suspend(if_ctx_t ctx) { - struct adapter *adapter = device_get_softc(dev); + struct adapter *adapter = iflib_get_softc(ctx); - EM_CORE_LOCK(adapter); - - em_release_manageability(adapter); + em_release_manageability(adapter); em_release_hw_control(adapter); - em_enable_wakeup(dev); - - EM_CORE_UNLOCK(adapter); - - return bus_generic_suspend(dev); + em_enable_wakeup(ctx); + return (0); } static int -em_resume(device_t dev) +em_if_resume(if_ctx_t ctx) { - struct adapter *adapter = device_get_softc(dev); - struct tx_ring *txr = adapter->tx_rings; - if_t ifp = adapter->ifp; + struct adapter *adapter = iflib_get_softc(ctx); - EM_CORE_LOCK(adapter); if (adapter->hw.mac.type == e1000_pch2lan) e1000_resume_workarounds_pchlan(&adapter->hw); - em_init_locked(adapter); + em_if_init(ctx); em_init_manageability(adapter); - if ((if_getflags(ifp) & IFF_UP) && - (if_getdrvflags(ifp) & IFF_DRV_RUNNING) && adapter->link_active) { - for (int i = 0; i < adapter->num_queues; i++, txr++) { - EM_TX_LOCK(txr); -#ifdef EM_MULTIQUEUE - if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr); -#else - if (!if_sendq_empty(ifp)) - em_start_locked(ifp, txr); -#endif - EM_TX_UNLOCK(txr); - } - } - EM_CORE_UNLOCK(adapter); - - return bus_generic_resume(dev); -} - - -#ifndef EM_MULTIQUEUE -static void -em_start_locked(if_t ifp, struct tx_ring *txr) -{ - struct adapter *adapter = if_getsoftc(ifp); - struct mbuf *m_head; - - EM_TX_LOCK_ASSERT(txr); - - if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != - IFF_DRV_RUNNING) - return; - - if (!adapter->link_active) - return; - - while (!if_sendq_empty(ifp)) { - /* Call cleanup if number of TX descriptors low */ - if (txr->tx_avail <= EM_TX_CLEANUP_THRESHOLD) - em_txeof(txr); - if (txr->tx_avail < EM_MAX_SCATTER) { - if_setdrvflagbits(ifp,IFF_DRV_OACTIVE, 0); - break; - } - m_head = if_dequeue(ifp); - if (m_head == NULL) - break; - /* - * Encapsulation can modify our pointer, and or make it - * NULL on failure. In that event, we can't requeue. - */ - if (em_xmit(txr, &m_head)) { - if (m_head == NULL) - break; - if_sendq_prepend(ifp, m_head); - break; - } - - /* Mark the queue as having work */ - if (txr->busy == EM_TX_IDLE) - txr->busy = EM_TX_BUSY; - - /* Send a copy of the frame to the BPF listener */ - ETHER_BPF_MTAP(ifp, m_head); - - } - - return; + return(0); } -static void -em_start(if_t ifp) -{ - struct adapter *adapter = if_getsoftc(ifp); - struct tx_ring *txr = adapter->tx_rings; - - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { - EM_TX_LOCK(txr); - em_start_locked(ifp, txr); - EM_TX_UNLOCK(txr); - } - return; -} -#else /* EM_MULTIQUEUE */ -/********************************************************************* - * Multiqueue Transmit routines - * - * em_mq_start is called by the stack to initiate a transmit. - * however, if busy the driver can queue the request rather - * than do an immediate send. It is this that is an advantage - * in this driver, rather than also having multiple tx queues. - **********************************************************************/ -/* -** Multiqueue capable stack interface -*/ static int -em_mq_start(if_t ifp, struct mbuf *m) +em_if_mtu_set(if_ctx_t ctx, uint32_t mtu) { - struct adapter *adapter = if_getsoftc(ifp); - struct tx_ring *txr = adapter->tx_rings; - unsigned int i, error; - - if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) - i = m->m_pkthdr.flowid % adapter->num_queues; - else - i = curcpu % adapter->num_queues; - - txr = &adapter->tx_rings[i]; - - error = drbr_enqueue(ifp, txr->br, m); - if (error) - return (error); + int max_frame_size; + struct adapter *adapter = iflib_get_softc(ctx); + if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx); - if (EM_TX_TRYLOCK(txr)) { - em_mq_start_locked(ifp, txr); - EM_TX_UNLOCK(txr); - } else - taskqueue_enqueue(txr->tq, &txr->tx_task); + IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); - return (0); -} - -static int -em_mq_start_locked(if_t ifp, struct tx_ring *txr) -{ - struct adapter *adapter = txr->adapter; - struct mbuf *next; - int err = 0, enq = 0; - - EM_TX_LOCK_ASSERT(txr); - - if (((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) || - adapter->link_active == 0) { - return (ENETDOWN); - } - - /* Process the queue */ - while ((next = drbr_peek(ifp, txr->br)) != NULL) { - if ((err = em_xmit(txr, &next)) != 0) { - if (next == NULL) { - /* It was freed, move forward */ - drbr_advance(ifp, txr->br); - } else { - /* - * Still have one left, it may not be - * the same since the transmit function - * may have changed it. - */ - drbr_putback(ifp, txr->br, next); - } - break; - } - drbr_advance(ifp, txr->br); - enq++; - if_inc_counter(ifp, IFCOUNTER_OBYTES, next->m_pkthdr.len); - if (next->m_flags & M_MCAST) - if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); - ETHER_BPF_MTAP(ifp, next); - if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) - break; - } - - /* Mark the queue as having work */ - if ((enq > 0) && (txr->busy == EM_TX_IDLE)) - txr->busy = EM_TX_BUSY; - - if (txr->tx_avail < EM_MAX_SCATTER) - em_txeof(txr); - if (txr->tx_avail < EM_MAX_SCATTER) { - if_setdrvflagbits(ifp, IFF_DRV_OACTIVE,0); - } - return (err); -} - -/* -** Flush all ring buffers -*/ -static void -em_qflush(if_t ifp) -{ - struct adapter *adapter = if_getsoftc(ifp); - struct tx_ring *txr = adapter->tx_rings; - struct mbuf *m; - - for (int i = 0; i < adapter->num_queues; i++, txr++) { - EM_TX_LOCK(txr); - while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) - m_freem(m); - EM_TX_UNLOCK(txr); - } - if_qflush(ifp); -} -#endif /* EM_MULTIQUEUE */ - -/********************************************************************* - * Ioctl entry point - * - * em_ioctl is called when the user wants to configure the - * interface. - * - * return 0 on success, positive on failure - **********************************************************************/ - -static int -em_ioctl(if_t ifp, u_long command, caddr_t data) -{ - struct adapter *adapter = if_getsoftc(ifp); - struct ifreq *ifr = (struct ifreq *)data; -#if defined(INET) || defined(INET6) - struct ifaddr *ifa = (struct ifaddr *)data; -#endif - bool avoid_reset = FALSE; - int error = 0; - - if (adapter->in_detach) - return (error); - - switch (command) { - case SIOCSIFADDR: -#ifdef INET - if (ifa->ifa_addr->sa_family == AF_INET) - avoid_reset = TRUE; -#endif -#ifdef INET6 - if (ifa->ifa_addr->sa_family == AF_INET6) - avoid_reset = TRUE; -#endif - /* - ** Calling init results in link renegotiation, - ** so we avoid doing it when possible. - */ - if (avoid_reset) { - if_setflagbits(ifp,IFF_UP,0); - if (!(if_getdrvflags(ifp)& IFF_DRV_RUNNING)) - em_init(adapter); -#ifdef INET - if (!(if_getflags(ifp) & IFF_NOARP)) - arp_ifinit(ifp, ifa); -#endif - } else - error = ether_ioctl(ifp, command, data); - break; - case SIOCSIFMTU: - { - int max_frame_size; - - IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); - - EM_CORE_LOCK(adapter); - switch (adapter->hw.mac.type) { - case e1000_82571: - case e1000_82572: - case e1000_ich9lan: - case e1000_ich10lan: - case e1000_pch2lan: - case e1000_pch_lpt: - case e1000_pch_spt: - case e1000_82574: - case e1000_82583: - case e1000_80003es2lan: /* 9K Jumbo Frame size */ - max_frame_size = 9234; - break; - case e1000_pchlan: - max_frame_size = 4096; - break; - /* Adapters that do not support jumbo frames */ - case e1000_ich8lan: - max_frame_size = ETHER_MAX_LEN; - break; - default: - max_frame_size = MAX_JUMBO_FRAME_SIZE; - } - if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - - ETHER_CRC_LEN) { - EM_CORE_UNLOCK(adapter); - error = EINVAL; - break; - } - - if_setmtu(ifp, ifr->ifr_mtu); - adapter->hw.mac.max_frame_size = - if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN; - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) - em_init_locked(adapter); - EM_CORE_UNLOCK(adapter); - break; - } - case SIOCSIFFLAGS: - IOCTL_DEBUGOUT("ioctl rcv'd:\ - SIOCSIFFLAGS (Set Interface Flags)"); - EM_CORE_LOCK(adapter); - if (if_getflags(ifp) & IFF_UP) { - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { - if ((if_getflags(ifp) ^ adapter->if_flags) & - (IFF_PROMISC | IFF_ALLMULTI)) { - em_disable_promisc(adapter); - em_set_promisc(adapter); - } - } else - em_init_locked(adapter); - } else - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) - em_stop(adapter); - adapter->if_flags = if_getflags(ifp); - EM_CORE_UNLOCK(adapter); - break; - case SIOCADDMULTI: - case SIOCDELMULTI: - IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { - EM_CORE_LOCK(adapter); - em_disable_intr(adapter); - em_set_multi(adapter); -#ifdef DEVICE_POLLING - if (!(if_getcapenable(ifp) & IFCAP_POLLING)) -#endif - em_enable_intr(adapter); - EM_CORE_UNLOCK(adapter); - } + switch (adapter->hw.mac.type) { + case e1000_82571: + case e1000_82572: + case e1000_ich9lan: + case e1000_ich10lan: + case e1000_pch2lan: + case e1000_pch_lpt: + case e1000_pch_spt: + case e1000_82574: + case e1000_82583: + case e1000_80003es2lan: + /* 9K Jumbo Frame size */ + max_frame_size = 9234; break; - case SIOCSIFMEDIA: - /* Check SOL/IDER usage */ - EM_CORE_LOCK(adapter); - if (e1000_check_reset_block(&adapter->hw)) { - EM_CORE_UNLOCK(adapter); - device_printf(adapter->dev, "Media change is" - " blocked due to SOL/IDER session.\n"); - break; - } - EM_CORE_UNLOCK(adapter); - /* falls thru */ - case SIOCGIFMEDIA: - IOCTL_DEBUGOUT("ioctl rcv'd: \ - SIOCxIFMEDIA (Get/Set Interface Media)"); - error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); + case e1000_pchlan: + max_frame_size = 4096; break; - case SIOCSIFCAP: - { - int mask, reinit; - - IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); - reinit = 0; - mask = ifr->ifr_reqcap ^ if_getcapenable(ifp); -#ifdef DEVICE_POLLING - if (mask & IFCAP_POLLING) { - if (ifr->ifr_reqcap & IFCAP_POLLING) { - error = ether_poll_register(em_poll, ifp); - if (error) - return (error); - EM_CORE_LOCK(adapter); - em_disable_intr(adapter); - if_setcapenablebit(ifp, IFCAP_POLLING, 0); - EM_CORE_UNLOCK(adapter); - } else { - error = ether_poll_deregister(ifp); - /* Enable interrupt even in error case */ - EM_CORE_LOCK(adapter); - em_enable_intr(adapter); - if_setcapenablebit(ifp, 0, IFCAP_POLLING); - EM_CORE_UNLOCK(adapter); - } - } -#endif - if (mask & IFCAP_HWCSUM) { - if_togglecapenable(ifp,IFCAP_HWCSUM); - reinit = 1; - } - if (mask & IFCAP_TSO4) { - if_togglecapenable(ifp,IFCAP_TSO4); - reinit = 1; - } - if (mask & IFCAP_VLAN_HWTAGGING) { - if_togglecapenable(ifp,IFCAP_VLAN_HWTAGGING); - reinit = 1; - } - if (mask & IFCAP_VLAN_HWFILTER) { - if_togglecapenable(ifp, IFCAP_VLAN_HWFILTER); - reinit = 1; - } - if (mask & IFCAP_VLAN_HWTSO) { - if_togglecapenable(ifp, IFCAP_VLAN_HWTSO); - reinit = 1; - } - if ((mask & IFCAP_WOL) && - (if_getcapabilities(ifp) & IFCAP_WOL) != 0) { - if (mask & IFCAP_WOL_MCAST) - if_togglecapenable(ifp, IFCAP_WOL_MCAST); - if (mask & IFCAP_WOL_MAGIC) - if_togglecapenable(ifp, IFCAP_WOL_MAGIC); - } - if (reinit && (if_getdrvflags(ifp) & IFF_DRV_RUNNING)) - em_init(adapter); - if_vlancap(ifp); + case e1000_82542: + case e1000_ich8lan: + /* Adapters that do not support jumbo frames */ + max_frame_size = ETHER_MAX_LEN; break; - } - default: - error = ether_ioctl(ifp, command, data); - break; + if (adapter->hw.mac.type >= igb_mac_min) + max_frame_size = 9234; + else /* lem */ + max_frame_size = MAX_JUMBO_FRAME_SIZE; + } + if (mtu > max_frame_size - ETHER_HDR_LEN - ETHER_CRC_LEN) { + return (EINVAL); } - return (error); + scctx->isc_max_frame_size = adapter->hw.mac.max_frame_size = + mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; + return (0); } - /********************************************************************* * Init entry point * @@ -1351,21 +1164,17 @@ em_ioctl(if_t ifp, u_long command, caddr_t data) **********************************************************************/ static void -em_init_locked(struct adapter *adapter) +em_if_init(if_ctx_t ctx) { - if_t ifp = adapter->ifp; - device_t dev = adapter->dev; - - INIT_DEBUGOUT("em_init: begin"); - - EM_CORE_LOCK_ASSERT(adapter); - - em_disable_intr(adapter); - callout_stop(&adapter->timer); + struct adapter *adapter = iflib_get_softc(ctx); + struct ifnet *ifp = iflib_get_ifp(ctx); + struct em_tx_queue *tx_que; + int i; + INIT_DEBUGOUT("em_if_init: begin"); /* Get the latest mac address, User can use a LAA */ - bcopy(if_getlladdr(adapter->ifp), adapter->hw.mac.addr, - ETHER_ADDR_LEN); + bcopy(if_getlladdr(ifp), adapter->hw.mac.addr, + ETHER_ADDR_LEN); /* Put the address into the Receive Address Array */ e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); @@ -1382,49 +1191,49 @@ em_init_locked(struct adapter *adapter) E1000_RAR_ENTRIES - 1); } + /* Initialize the hardware */ - em_reset(adapter); - em_update_link_status(adapter); + em_reset(ctx); + em_if_update_admin_status(ctx); + + for (i = 0, tx_que = adapter->tx_queues; i < adapter->tx_num_queues; i++, tx_que++) { + struct tx_ring *txr = &tx_que->txr; + + txr->tx_rs_cidx = txr->tx_rs_pidx = txr->tx_cidx_processed = 0; + } /* Setup VLAN support, basic and offload if available */ E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); - /* Set hardware offload abilities */ - if_clearhwassist(ifp); - if (if_getcapenable(ifp) & IFCAP_TXCSUM) - if_sethwassistbits(ifp, CSUM_TCP | CSUM_UDP, 0); - - if (if_getcapenable(ifp) & IFCAP_TSO4) - if_sethwassistbits(ifp, CSUM_TSO, 0); + /* Clear bad data from Rx FIFOs */ + if (adapter->hw.mac.type >= igb_mac_min) + e1000_rx_fifo_flush_82575(&adapter->hw); /* Configure for OS presence */ em_init_manageability(adapter); /* Prepare transmit descriptors and buffers */ - em_setup_transmit_structures(adapter); - em_initialize_transmit_unit(adapter); + em_initialize_transmit_unit(ctx); /* Setup Multicast table */ - em_set_multi(adapter); + em_if_multi_set(ctx); /* - ** Figure out the desired mbuf - ** pool for doing jumbos - */ + * Figure out the desired mbuf + * pool for doing jumbos + */ if (adapter->hw.mac.max_frame_size <= 2048) adapter->rx_mbuf_sz = MCLBYTES; +#ifndef CONTIGMALLOC_WORKS + else + adapter->rx_mbuf_sz = MJUMPAGESIZE; +#else else if (adapter->hw.mac.max_frame_size <= 4096) adapter->rx_mbuf_sz = MJUMPAGESIZE; else adapter->rx_mbuf_sz = MJUM9BYTES; - - /* Prepare receive descriptors and buffers */ - if (em_setup_receive_structures(adapter)) { - device_printf(dev, "Could not setup receive structures\n"); - em_stop(adapter); - return; - } - em_initialize_receive_unit(adapter); +#endif + em_initialize_receive_unit(ctx); /* Use real VLAN Filter support? */ if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) { @@ -1440,123 +1249,59 @@ em_init_locked(struct adapter *adapter) } /* Don't lose promiscuous settings */ - em_set_promisc(adapter); - - /* Set the interface as ACTIVE */ - if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); - - callout_reset(&adapter->timer, hz, em_local_timer, adapter); + em_if_set_promisc(ctx, IFF_PROMISC); e1000_clear_hw_cntrs_base_generic(&adapter->hw); /* MSI/X configuration for 82574 */ if (adapter->hw.mac.type == e1000_82574) { - int tmp; - tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); + int tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); + tmp |= E1000_CTRL_EXT_PBA_CLR; E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp); /* Set the IVAR - interrupt vector routing. */ E1000_WRITE_REG(&adapter->hw, E1000_IVAR, adapter->ivars); - } + } else if (adapter->intr_type == IFLIB_INTR_MSIX) /* Set up queue routing */ + igb_configure_queues(adapter); -#ifdef DEVICE_POLLING - /* - * Only enable interrupts if we are not polling, make sure - * they are off otherwise. - */ - if (if_getcapenable(ifp) & IFCAP_POLLING) - em_disable_intr(adapter); - else -#endif /* DEVICE_POLLING */ - em_enable_intr(adapter); + /* this clears any pending interrupts */ + E1000_READ_REG(&adapter->hw, E1000_ICR); + E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC); /* AMT based hardware can now take control from firmware */ if (adapter->has_manage && adapter->has_amt) em_get_hw_control(adapter); -} - -static void -em_init(void *arg) -{ - struct adapter *adapter = arg; - - EM_CORE_LOCK(adapter); - em_init_locked(adapter); - EM_CORE_UNLOCK(adapter); -} - - -#ifdef DEVICE_POLLING -/********************************************************************* - * - * Legacy polling routine: note this only works with single queue - * - *********************************************************************/ -static int -em_poll(if_t ifp, enum poll_cmd cmd, int count) -{ - struct adapter *adapter = if_getsoftc(ifp); - struct tx_ring *txr = adapter->tx_rings; - struct rx_ring *rxr = adapter->rx_rings; - u32 reg_icr; - int rx_done; - - EM_CORE_LOCK(adapter); - if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) { - EM_CORE_UNLOCK(adapter); - return (0); - } - if (cmd == POLL_AND_CHECK_STATUS) { - reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); - if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { - callout_stop(&adapter->timer); - adapter->hw.mac.get_link_status = 1; - em_update_link_status(adapter); - callout_reset(&adapter->timer, hz, - em_local_timer, adapter); - } + /* Set Energy Efficient Ethernet */ + if (adapter->hw.mac.type >= igb_mac_min && + adapter->hw.phy.media_type == e1000_media_type_copper) { + if (adapter->hw.mac.type == e1000_i354) + e1000_set_eee_i354(&adapter->hw, TRUE, TRUE); + else + e1000_set_eee_i350(&adapter->hw, TRUE, TRUE); } - EM_CORE_UNLOCK(adapter); - - em_rxeof(rxr, count, &rx_done); - - EM_TX_LOCK(txr); - em_txeof(txr); -#ifdef EM_MULTIQUEUE - if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr); -#else - if (!if_sendq_empty(ifp)) - em_start_locked(ifp, txr); -#endif - EM_TX_UNLOCK(txr); - - return (rx_done); } -#endif /* DEVICE_POLLING */ - /********************************************************************* * - * Fast Legacy/MSI Combined Interrupt Service routine + * Fast Legacy/MSI Combined Interrupt Service routine * *********************************************************************/ -static int -em_irq_fast(void *arg) +int +em_intr(void *arg) { - struct adapter *adapter = arg; - if_t ifp; - u32 reg_icr; - - ifp = adapter->ifp; + struct adapter *adapter = arg; + if_ctx_t ctx = adapter->ctx; + u32 reg_icr; reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); - /* Hot eject? */ + if (adapter->intr_type != IFLIB_INTR_LEGACY) + goto skip_stray; + /* Hot eject? */ if (reg_icr == 0xffffffff) return FILTER_STRAY; - /* Definitely not our interrupt. */ + /* Definitely not our interrupt. */ if (reg_icr == 0x0) return FILTER_STRAY; @@ -1568,80 +1313,67 @@ em_irq_fast(void *arg) (reg_icr & E1000_ICR_INT_ASSERTED) == 0) return FILTER_STRAY; - em_disable_intr(adapter); - taskqueue_enqueue(adapter->tq, &adapter->que_task); - +skip_stray: /* Link status change */ if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { adapter->hw.mac.get_link_status = 1; - taskqueue_enqueue(taskqueue_fast, &adapter->link_task); + iflib_admin_intr_deferred(ctx); } if (reg_icr & E1000_ICR_RXO) adapter->rx_overruns++; - return FILTER_HANDLED; + + return (FILTER_SCHEDULE_THREAD); } -/* Combined RX/TX handler, used by Legacy and MSI */ static void -em_handle_que(void *context, int pending) +igb_rx_enable_queue(struct adapter *adapter, struct em_rx_queue *rxq) { - struct adapter *adapter = context; - if_t ifp = adapter->ifp; - struct tx_ring *txr = adapter->tx_rings; - struct rx_ring *rxr = adapter->rx_rings; - - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { - bool more = em_rxeof(rxr, adapter->rx_process_limit, NULL); - - EM_TX_LOCK(txr); - em_txeof(txr); -#ifdef EM_MULTIQUEUE - if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr); -#else - if (!if_sendq_empty(ifp)) - em_start_locked(ifp, txr); -#endif - EM_TX_UNLOCK(txr); - if (more) { - taskqueue_enqueue(adapter->tq, &adapter->que_task); - return; - } - } + E1000_WRITE_REG(&adapter->hw, E1000_EIMS, rxq->eims); +} - em_enable_intr(adapter); - return; +static void +em_rx_enable_queue(struct adapter *adapter, struct em_rx_queue *rxq) +{ + E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxq->eims); } +static void +igb_tx_enable_queue(struct adapter *adapter, struct em_tx_queue *txq) +{ + E1000_WRITE_REG(&adapter->hw, E1000_EIMS, txq->eims); +} -/********************************************************************* - * - * MSIX Interrupt Service Routines - * - **********************************************************************/ static void -em_msix_tx(void *arg) +em_tx_enable_queue(struct adapter *adapter, struct em_tx_queue *txq) { - struct tx_ring *txr = arg; - struct adapter *adapter = txr->adapter; - if_t ifp = adapter->ifp; - - ++txr->tx_irq; - EM_TX_LOCK(txr); - em_txeof(txr); -#ifdef EM_MULTIQUEUE - if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr); -#else - if (!if_sendq_empty(ifp)) - em_start_locked(ifp, txr); -#endif + E1000_WRITE_REG(&adapter->hw, E1000_IMS, txq->eims); +} - /* Reenable this interrupt */ - E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims); - EM_TX_UNLOCK(txr); - return; +static int +em_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) +{ + struct adapter *adapter = iflib_get_softc(ctx); + struct em_rx_queue *rxq = &adapter->rx_queues[rxqid]; + + if (adapter->hw.mac.type >= igb_mac_min) + igb_rx_enable_queue(adapter, rxq); + else + em_rx_enable_queue(adapter, rxq); + return (0); +} + +static int +em_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid) +{ + struct adapter *adapter = iflib_get_softc(ctx); + struct em_tx_queue *txq = &adapter->tx_queues[txqid]; + + if (adapter->hw.mac.type >= igb_mac_min) + igb_tx_enable_queue(adapter, txq); + else + em_tx_enable_queue(adapter, txq); + return (0); } /********************************************************************* @@ -1649,25 +1381,14 @@ em_msix_tx(void *arg) * MSIX RX Interrupt Service routine * **********************************************************************/ - -static void -em_msix_rx(void *arg) +static int +em_msix_que(void *arg) { - struct rx_ring *rxr = arg; - struct adapter *adapter = rxr->adapter; - bool more; + struct em_rx_queue *que = arg; - ++rxr->rx_irq; - if (!(if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING)) - return; - more = em_rxeof(rxr, adapter->rx_process_limit, NULL); - if (more) - taskqueue_enqueue(rxr->tq, &rxr->rx_task); - else { - /* Reenable this interrupt */ - E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims); - } - return; + ++que->irqs; + + return (FILTER_SCHEDULE_THREAD); } /********************************************************************* @@ -1675,103 +1396,50 @@ em_msix_rx(void *arg) * MSIX Link Fast Interrupt Service routine * **********************************************************************/ -static void +static int em_msix_link(void *arg) { - struct adapter *adapter = arg; - u32 reg_icr; + struct adapter *adapter = arg; + u32 reg_icr; ++adapter->link_irq; + MPASS(adapter->hw.back != NULL); reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); if (reg_icr & E1000_ICR_RXO) adapter->rx_overruns++; if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { - adapter->hw.mac.get_link_status = 1; - em_handle_link(adapter, 0); - } else + em_handle_link(adapter->ctx); + } else { E1000_WRITE_REG(&adapter->hw, E1000_IMS, - EM_MSIX_LINK | E1000_IMS_LSC); + EM_MSIX_LINK | E1000_IMS_LSC); + if (adapter->hw.mac.type >= igb_mac_min) + E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask); + } + /* - ** Because we must read the ICR for this interrupt - ** it may clear other causes using autoclear, for - ** this reason we simply create a soft interrupt - ** for all these vectors. - */ - if (reg_icr) { + * Because we must read the ICR for this interrupt + * it may clear other causes using autoclear, for + * this reason we simply create a soft interrupt + * for all these vectors. + */ + if (reg_icr && adapter->hw.mac.type < igb_mac_min) { E1000_WRITE_REG(&adapter->hw, E1000_ICS, adapter->ims); } - return; -} - -static void -em_handle_rx(void *context, int pending) -{ - struct rx_ring *rxr = context; - struct adapter *adapter = rxr->adapter; - bool more; - more = em_rxeof(rxr, adapter->rx_process_limit, NULL); - if (more) - taskqueue_enqueue(rxr->tq, &rxr->rx_task); - else { - /* Reenable this interrupt */ - E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims); - } + return (FILTER_HANDLED); } static void -em_handle_tx(void *context, int pending) +em_handle_link(void *context) { - struct tx_ring *txr = context; - struct adapter *adapter = txr->adapter; - if_t ifp = adapter->ifp; - - EM_TX_LOCK(txr); - em_txeof(txr); -#ifdef EM_MULTIQUEUE - if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr); -#else - if (!if_sendq_empty(ifp)) - em_start_locked(ifp, txr); -#endif - E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims); - EM_TX_UNLOCK(txr); -} + if_ctx_t ctx = context; + struct adapter *adapter = iflib_get_softc(ctx); -static void -em_handle_link(void *context, int pending) -{ - struct adapter *adapter = context; - struct tx_ring *txr = adapter->tx_rings; - if_t ifp = adapter->ifp; - - if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) - return; - - EM_CORE_LOCK(adapter); - callout_stop(&adapter->timer); - em_update_link_status(adapter); - callout_reset(&adapter->timer, hz, em_local_timer, adapter); - E1000_WRITE_REG(&adapter->hw, E1000_IMS, - EM_MSIX_LINK | E1000_IMS_LSC); - if (adapter->link_active) { - for (int i = 0; i < adapter->num_queues; i++, txr++) { - EM_TX_LOCK(txr); -#ifdef EM_MULTIQUEUE - if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr); -#else - if (if_sendq_empty(ifp)) - em_start_locked(ifp, txr); -#endif - EM_TX_UNLOCK(txr); - } - } - EM_CORE_UNLOCK(adapter); + adapter->hw.mac.get_link_status = 1; + iflib_admin_intr_deferred(ctx); } @@ -1784,21 +1452,19 @@ em_handle_link(void *context, int pending) * **********************************************************************/ static void -em_media_status(if_t ifp, struct ifmediareq *ifmr) +em_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) { - struct adapter *adapter = if_getsoftc(ifp); + struct adapter *adapter = iflib_get_softc(ctx); u_char fiber_type = IFM_1000_SX; - INIT_DEBUGOUT("em_media_status: begin"); + INIT_DEBUGOUT("em_if_media_status: begin"); - EM_CORE_LOCK(adapter); - em_update_link_status(adapter); + iflib_admin_intr_deferred(ctx); ifmr->ifm_status = IFM_AVALID; ifmr->ifm_active = IFM_ETHER; if (!adapter->link_active) { - EM_CORE_UNLOCK(adapter); return; } @@ -1806,6 +1472,8 @@ em_media_status(if_t ifp, struct ifmediareq *ifmr) if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { + if (adapter->hw.mac.type == e1000_82545) + fiber_type = IFM_1000_LX; ifmr->ifm_active |= fiber_type | IFM_FDX; } else { switch (adapter->link_speed) { @@ -1824,7 +1492,6 @@ em_media_status(if_t ifp, struct ifmediareq *ifmr) else ifmr->ifm_active |= IFM_HDX; } - EM_CORE_UNLOCK(adapter); } /********************************************************************* @@ -1836,17 +1503,16 @@ em_media_status(if_t ifp, struct ifmediareq *ifmr) * **********************************************************************/ static int -em_media_change(if_t ifp) +em_if_media_change(if_ctx_t ctx) { - struct adapter *adapter = if_getsoftc(ifp); - struct ifmedia *ifm = &adapter->media; + struct adapter *adapter = iflib_get_softc(ctx); + struct ifmedia *ifm = iflib_get_media(ctx); - INIT_DEBUGOUT("em_media_change: begin"); + INIT_DEBUGOUT("em_if_media_change: begin"); if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) return (EINVAL); - EM_CORE_LOCK(adapter); switch (IFM_SUBTYPE(ifm->ifm_media)) { case IFM_AUTO: adapter->hw.mac.autoneg = DO_AUTO_NEG; @@ -1878,361 +1544,45 @@ em_media_change(if_t ifp) device_printf(adapter->dev, "Unsupported media type\n"); } - em_init_locked(adapter); - EM_CORE_UNLOCK(adapter); + em_if_init(ctx); return (0); } -/********************************************************************* - * - * This routine maps the mbufs to tx descriptors. - * - * return 0 on success, positive on failure - **********************************************************************/ - static int -em_xmit(struct tx_ring *txr, struct mbuf **m_headp) +em_if_set_promisc(if_ctx_t ctx, int flags) { - struct adapter *adapter = txr->adapter; - bus_dma_segment_t segs[EM_MAX_SCATTER]; - bus_dmamap_t map; - struct em_txbuffer *tx_buffer, *tx_buffer_mapped; - struct e1000_tx_desc *ctxd = NULL; - struct mbuf *m_head; - struct ether_header *eh; - struct ip *ip = NULL; - struct tcphdr *tp = NULL; - u32 txd_upper = 0, txd_lower = 0; - int ip_off, poff; - int nsegs, i, j, first, last = 0; - int error; - bool do_tso, tso_desc, remap = TRUE; - - m_head = *m_headp; - do_tso = (m_head->m_pkthdr.csum_flags & CSUM_TSO); - tso_desc = FALSE; - ip_off = poff = 0; - - /* - * Intel recommends entire IP/TCP header length reside in a single - * buffer. If multiple descriptors are used to describe the IP and - * TCP header, each descriptor should describe one or more - * complete headers; descriptors referencing only parts of headers - * are not supported. If all layer headers are not coalesced into - * a single buffer, each buffer should not cross a 4KB boundary, - * or be larger than the maximum read request size. - * Controller also requires modifing IP/TCP header to make TSO work - * so we firstly get a writable mbuf chain then coalesce ethernet/ - * IP/TCP header into a single buffer to meet the requirement of - * controller. This also simplifies IP/TCP/UDP checksum offloading - * which also has similar restrictions. - */ - if (do_tso || m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) { - if (do_tso || (m_head->m_next != NULL && - m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)) { - if (M_WRITABLE(*m_headp) == 0) { - m_head = m_dup(*m_headp, M_NOWAIT); - m_freem(*m_headp); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); - } - *m_headp = m_head; - } - } - /* - * XXX - * Assume IPv4, we don't have TSO/checksum offload support - * for IPv6 yet. - */ - ip_off = sizeof(struct ether_header); - if (m_head->m_len < ip_off) { - m_head = m_pullup(m_head, ip_off); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); - } - } - eh = mtod(m_head, struct ether_header *); - if (eh->ether_type == htons(ETHERTYPE_VLAN)) { - ip_off = sizeof(struct ether_vlan_header); - if (m_head->m_len < ip_off) { - m_head = m_pullup(m_head, ip_off); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); - } - } - } - if (m_head->m_len < ip_off + sizeof(struct ip)) { - m_head = m_pullup(m_head, ip_off + sizeof(struct ip)); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); - } - } - ip = (struct ip *)(mtod(m_head, char *) + ip_off); - poff = ip_off + (ip->ip_hl << 2); - - if (do_tso || (m_head->m_pkthdr.csum_flags & CSUM_TCP)) { - if (m_head->m_len < poff + sizeof(struct tcphdr)) { - m_head = m_pullup(m_head, poff + - sizeof(struct tcphdr)); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); - } - } - tp = (struct tcphdr *)(mtod(m_head, char *) + poff); - /* - * TSO workaround: - * pull 4 more bytes of data into it. - */ - if (m_head->m_len < poff + (tp->th_off << 2)) { - m_head = m_pullup(m_head, poff + - (tp->th_off << 2) + - TSO_WORKAROUND); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); - } - } - ip = (struct ip *)(mtod(m_head, char *) + ip_off); - tp = (struct tcphdr *)(mtod(m_head, char *) + poff); - if (do_tso) { - ip->ip_len = htons(m_head->m_pkthdr.tso_segsz + - (ip->ip_hl << 2) + - (tp->th_off << 2)); - ip->ip_sum = 0; - /* - * The pseudo TCP checksum does not include TCP - * payload length so driver should recompute - * the checksum here what hardware expect to - * see. This is adherence of Microsoft's Large - * Send specification. - */ - tp->th_sum = in_pseudo(ip->ip_src.s_addr, - ip->ip_dst.s_addr, htons(IPPROTO_TCP)); - } - } else if (m_head->m_pkthdr.csum_flags & CSUM_UDP) { - if (m_head->m_len < poff + sizeof(struct udphdr)) { - m_head = m_pullup(m_head, poff + - sizeof(struct udphdr)); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); - } - } - ip = (struct ip *)(mtod(m_head, char *) + ip_off); - } - *m_headp = m_head; - } - - /* - * Map the packet for DMA - * - * Capture the first descriptor index, - * this descriptor will have the index - * of the EOP which is the only one that - * now gets a DONE bit writeback. - */ - first = txr->next_avail_desc; - tx_buffer = &txr->tx_buffers[first]; - tx_buffer_mapped = tx_buffer; - map = tx_buffer->map; - -retry: - error = bus_dmamap_load_mbuf_sg(txr->txtag, map, - *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); - - /* - * There are two types of errors we can (try) to handle: - * - EFBIG means the mbuf chain was too long and bus_dma ran - * out of segments. Defragment the mbuf chain and try again. - * - ENOMEM means bus_dma could not obtain enough bounce buffers - * at this point in time. Defer sending and try again later. - * All other errors, in particular EINVAL, are fatal and prevent the - * mbuf chain from ever going through. Drop it and report error. - */ - if (error == EFBIG && remap) { - struct mbuf *m; - - m = m_collapse(*m_headp, M_NOWAIT, EM_MAX_SCATTER); - if (m == NULL) { - adapter->mbuf_defrag_failed++; - m_freem(*m_headp); - *m_headp = NULL; - return (ENOBUFS); - } - *m_headp = m; - - /* Try it again, but only once */ - remap = FALSE; - goto retry; - } else if (error != 0) { - adapter->no_tx_dma_setup++; - m_freem(*m_headp); - *m_headp = NULL; - return (error); - } - - /* - * TSO Hardware workaround, if this packet is not - * TSO, and is only a single descriptor long, and - * it follows a TSO burst, then we need to add a - * sentinel descriptor to prevent premature writeback. - */ - if ((!do_tso) && (txr->tx_tso == TRUE)) { - if (nsegs == 1) - tso_desc = TRUE; - txr->tx_tso = FALSE; - } - - if (txr->tx_avail < (nsegs + EM_MAX_SCATTER)) { - txr->no_desc_avail++; - bus_dmamap_unload(txr->txtag, map); - return (ENOBUFS); - } - m_head = *m_headp; - - /* Do hardware assists */ - if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { - em_tso_setup(txr, m_head, ip_off, ip, tp, - &txd_upper, &txd_lower); - /* we need to make a final sentinel transmit desc */ - tso_desc = TRUE; - } else if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) - em_transmit_checksum_setup(txr, m_head, - ip_off, ip, &txd_upper, &txd_lower); - - if (m_head->m_flags & M_VLANTAG) { - /* Set the vlan id. */ - txd_upper |= htole16(if_getvtag(m_head)) << 16; - /* Tell hardware to add tag */ - txd_lower |= htole32(E1000_TXD_CMD_VLE); - } - - i = txr->next_avail_desc; - - /* Set up our transmit descriptors */ - for (j = 0; j < nsegs; j++) { - bus_size_t seg_len; - bus_addr_t seg_addr; - - tx_buffer = &txr->tx_buffers[i]; - ctxd = &txr->tx_base[i]; - seg_addr = segs[j].ds_addr; - seg_len = segs[j].ds_len; - /* - ** TSO Workaround: - ** If this is the last descriptor, we want to - ** split it so we have a small final sentinel - */ - if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) { - seg_len -= TSO_WORKAROUND; - ctxd->buffer_addr = htole64(seg_addr); - ctxd->lower.data = htole32( - adapter->txd_cmd | txd_lower | seg_len); - ctxd->upper.data = htole32(txd_upper); - if (++i == adapter->num_tx_desc) - i = 0; - - /* Now make the sentinel */ - txr->tx_avail--; - ctxd = &txr->tx_base[i]; - tx_buffer = &txr->tx_buffers[i]; - ctxd->buffer_addr = - htole64(seg_addr + seg_len); - ctxd->lower.data = htole32( - adapter->txd_cmd | txd_lower | TSO_WORKAROUND); - ctxd->upper.data = - htole32(txd_upper); - last = i; - if (++i == adapter->num_tx_desc) - i = 0; - } else { - ctxd->buffer_addr = htole64(seg_addr); - ctxd->lower.data = htole32( - adapter->txd_cmd | txd_lower | seg_len); - ctxd->upper.data = htole32(txd_upper); - last = i; - if (++i == adapter->num_tx_desc) - i = 0; - } - tx_buffer->m_head = NULL; - tx_buffer->next_eop = -1; - } - - txr->next_avail_desc = i; - txr->tx_avail -= nsegs; + struct adapter *adapter = iflib_get_softc(ctx); + u32 reg_rctl; - tx_buffer->m_head = m_head; - /* - ** Here we swap the map so the last descriptor, - ** which gets the completion interrupt has the - ** real map, and the first descriptor gets the - ** unused map from this descriptor. - */ - tx_buffer_mapped->map = tx_buffer->map; - tx_buffer->map = map; - bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); - - /* - * Last Descriptor of Packet - * needs End Of Packet (EOP) - * and Report Status (RS) - */ - ctxd->lower.data |= - htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS); - /* - * Keep track in the first buffer which - * descriptor will be written back - */ - tx_buffer = &txr->tx_buffers[first]; - tx_buffer->next_eop = last; - - /* - * Advance the Transmit Descriptor Tail (TDT), this tells the E1000 - * that this frame is available to transmit. - */ - bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i); - - return (0); -} - -static void -em_set_promisc(struct adapter *adapter) -{ - if_t ifp = adapter->ifp; - u32 reg_rctl; + em_disable_promisc(ctx); reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); - if (if_getflags(ifp) & IFF_PROMISC) { + if (flags & IFF_PROMISC) { reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); /* Turn this on if you want to see bad packets */ if (em_debug_sbp) reg_rctl |= E1000_RCTL_SBP; E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); - } else if (if_getflags(ifp) & IFF_ALLMULTI) { + } else if (flags & IFF_ALLMULTI) { reg_rctl |= E1000_RCTL_MPE; reg_rctl &= ~E1000_RCTL_UPE; E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); } + return (0); } static void -em_disable_promisc(struct adapter *adapter) +em_disable_promisc(if_ctx_t ctx) { - if_t ifp = adapter->ifp; - u32 reg_rctl; - int mcnt = 0; + struct adapter *adapter = iflib_get_softc(ctx); + struct ifnet *ifp = iflib_get_ifp(ctx); + u32 reg_rctl; + int mcnt = 0; reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); - reg_rctl &= (~E1000_RCTL_UPE); + reg_rctl &= (~E1000_RCTL_UPE); if (if_getflags(ifp) & IFF_ALLMULTI) mcnt = MAX_NUM_MULTICAST_ADDRESSES; else @@ -2253,9 +1603,10 @@ em_disable_promisc(struct adapter *adapter) **********************************************************************/ static void -em_set_multi(struct adapter *adapter) +em_if_multi_set(if_ctx_t ctx) { - if_t ifp = adapter->ifp; + struct adapter *adapter = iflib_get_softc(ctx); + struct ifnet *ifp = iflib_get_ifp(ctx); u32 reg_rctl = 0; u8 *mta; /* Multicast array memory */ int mcnt = 0; @@ -2265,7 +1616,7 @@ em_set_multi(struct adapter *adapter) mta = adapter->mta; bzero(mta, sizeof(u8) * ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES); - if (adapter->hw.mac.type == e1000_82542 && + if (adapter->hw.mac.type == e1000_82542 && adapter->hw.revision_id == E1000_REVISION_2) { reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) @@ -2284,7 +1635,7 @@ em_set_multi(struct adapter *adapter) } else e1000_update_mc_addr_list(&adapter->hw, mta, mcnt); - if (adapter->hw.mac.type == e1000_82542 && + if (adapter->hw.mac.type == e1000_82542 && adapter->hw.revision_id == E1000_REVISION_2) { reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); reg_rctl &= ~E1000_RCTL_RST; @@ -2304,17 +1655,17 @@ em_set_multi(struct adapter *adapter) **********************************************************************/ static void -em_local_timer(void *arg) +em_if_timer(if_ctx_t ctx, uint16_t qid) { - struct adapter *adapter = arg; - if_t ifp = adapter->ifp; - struct tx_ring *txr = adapter->tx_rings; - struct rx_ring *rxr = adapter->rx_rings; - u32 trigger = 0; + struct adapter *adapter = iflib_get_softc(ctx); + struct em_rx_queue *que; + int i; + int trigger = 0; - EM_CORE_LOCK_ASSERT(adapter); + if (qid != 0) + return; - em_update_link_status(adapter); + em_if_update_admin_status(ctx); em_update_stats_counters(adapter); /* Reset LAA into RAR[0] on 82571 */ @@ -2322,53 +1673,26 @@ em_local_timer(void *arg) e1000_get_laa_state_82571(&adapter->hw)) e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); + if (adapter->hw.mac.type < em_mac_min) + lem_smartspeed(adapter); + /* Mask to use in the irq trigger */ - if (adapter->msix_mem) { - for (int i = 0; i < adapter->num_queues; i++, rxr++) - trigger |= rxr->ims; - rxr = adapter->rx_rings; - } else + if (adapter->intr_type == IFLIB_INTR_MSIX) { + for (i = 0, que = adapter->rx_queues; i < adapter->rx_num_queues; i++, que++) + trigger |= que->eims; + } else { trigger = E1000_ICS_RXDMT0; - - /* - ** Check on the state of the TX queue(s), this - ** can be done without the lock because its RO - ** and the HUNG state will be static if set. - */ - for (int i = 0; i < adapter->num_queues; i++, txr++) { - if (txr->busy == EM_TX_HUNG) - goto hung; - if (txr->busy >= EM_TX_MAXTRIES) - txr->busy = EM_TX_HUNG; - /* Schedule a TX tasklet if needed */ - if (txr->tx_avail <= EM_MAX_SCATTER) - taskqueue_enqueue(txr->tq, &txr->tx_task); } - - callout_reset(&adapter->timer, hz, em_local_timer, adapter); -#ifndef DEVICE_POLLING - /* Trigger an RX interrupt to guarantee mbuf refresh */ - E1000_WRITE_REG(&adapter->hw, E1000_ICS, trigger); -#endif - return; -hung: - /* Looks like we're hung */ - device_printf(adapter->dev, "Watchdog timeout Queue[%d]-- resetting\n", - txr->me); - em_print_debug_info(adapter); - if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); - adapter->watchdog_events++; - em_init_locked(adapter); } static void -em_update_link_status(struct adapter *adapter) +em_if_update_admin_status(if_ctx_t ctx) { + struct adapter *adapter = iflib_get_softc(ctx); struct e1000_hw *hw = &adapter->hw; - if_t ifp = adapter->ifp; - device_t dev = adapter->dev; - struct tx_ring *txr = adapter->tx_rings; + struct ifnet *ifp = iflib_get_ifp(ctx); + device_t dev = iflib_get_dev(ctx); u32 link_check = 0; /* Get the cached link value or read phy for real */ @@ -2382,13 +1706,14 @@ em_update_link_status(struct adapter *adapter) link_check = !hw->mac.get_link_status; if (link_check) /* ESB2 fix */ e1000_cfg_on_link_up(hw); - } else + } else { link_check = TRUE; + } break; case e1000_media_type_fiber: e1000_check_for_link(hw); link_check = (E1000_READ_REG(hw, E1000_STATUS) & - E1000_STATUS_LU); + E1000_STATUS_LU); break; case e1000_media_type_internal_serdes: e1000_check_for_link(hw); @@ -2403,18 +1728,6 @@ em_update_link_status(struct adapter *adapter) if (link_check && (adapter->link_active == 0)) { e1000_get_speed_and_duplex(hw, &adapter->link_speed, &adapter->link_duplex); - /* - ** There have proven to be problems with TSO when not - ** at full gigabit speed, so disable the assist automatically - ** when at lower speeds. -jfv - */ - if (adapter->link_speed != SPEED_1000) { - if_sethwassistbits(ifp, 0, CSUM_TSO); - if_setcapenablebit(ifp, 0, IFCAP_TSO4); - if_setcapabilitiesbit(ifp, 0, IFCAP_TSO4); - - } - /* Check if we must disable SPEED_MODE bit on PCI-E */ if ((adapter->link_speed != SPEED_1000) && ((hw->mac.type == e1000_82571) || @@ -2432,7 +1745,8 @@ em_update_link_status(struct adapter *adapter) adapter->link_active = 1; adapter->smartspeed = 0; if_setbaudrate(ifp, adapter->link_speed * 1000000); - if_link_state_change(ifp, LINK_STATE_UP); + iflib_link_state_change(ctx, LINK_STATE_UP, ifp->if_baudrate); + printf("Link state changed to up\n"); } else if (!link_check && (adapter->link_active == 1)) { if_setbaudrate(ifp, 0); adapter->link_speed = 0; @@ -2440,11 +1754,11 @@ em_update_link_status(struct adapter *adapter) if (bootverbose) device_printf(dev, "Link is Down\n"); adapter->link_active = 0; - /* Link down, disable hang detection */ - for (int i = 0; i < adapter->num_queues; i++, txr++) - txr->busy = EM_TX_IDLE; - if_link_state_change(ifp, LINK_STATE_DOWN); + iflib_link_state_change(ctx, LINK_STATE_DOWN, ifp->if_baudrate); + printf("link state changed to down\n"); } + + E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_LINK | E1000_IMS_LSC); } /********************************************************************* @@ -2457,35 +1771,15 @@ em_update_link_status(struct adapter *adapter) **********************************************************************/ static void -em_stop(void *arg) +em_if_stop(if_ctx_t ctx) { - struct adapter *adapter = arg; - if_t ifp = adapter->ifp; - struct tx_ring *txr = adapter->tx_rings; - - EM_CORE_LOCK_ASSERT(adapter); + struct adapter *adapter = iflib_get_softc(ctx); INIT_DEBUGOUT("em_stop: begin"); - em_disable_intr(adapter); - callout_stop(&adapter->timer); - - /* Tell the stack that the interface is no longer active */ - if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); - - /* Disarm Hang Detection. */ - for (int i = 0; i < adapter->num_queues; i++, txr++) { - EM_TX_LOCK(txr); - txr->busy = EM_TX_IDLE; - EM_TX_UNLOCK(txr); - } - - /* I219 needs some special flushing to avoid hangs */ - if (adapter->hw.mac.type == e1000_pch_spt) - em_flush_desc_rings(adapter); - e1000_reset_hw(&adapter->hw); - E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); + if (adapter->hw.mac.type >= e1000_82544) + E1000_WRITE_REG(&adapter->hw, E1000_WUFC, 0); e1000_led_off(&adapter->hw); e1000_cleanup_led(&adapter->hw); @@ -2498,12 +1792,12 @@ em_stop(void *arg) * **********************************************************************/ static void -em_identify_hardware(struct adapter *adapter) +em_identify_hardware(if_ctx_t ctx) { - device_t dev = adapter->dev; + device_t dev = iflib_get_dev(ctx); + struct adapter *adapter = iflib_get_softc(ctx); /* Make sure our PCI config space has the necessary stuff set */ - pci_enable_busmaster(dev); adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); /* Save off the information about this board */ @@ -2523,10 +1817,11 @@ em_identify_hardware(struct adapter *adapter) } static int -em_allocate_pci_resources(struct adapter *adapter) +em_allocate_pci_resources(if_ctx_t ctx) { - device_t dev = adapter->dev; - int rid; + struct adapter *adapter = iflib_get_softc(ctx); + device_t dev = iflib_get_dev(ctx); + int rid, val; rid = PCIR_BAR(0); adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, @@ -2535,498 +1830,398 @@ em_allocate_pci_resources(struct adapter *adapter) device_printf(dev, "Unable to allocate bus resource: memory\n"); return (ENXIO); } - adapter->osdep.mem_bus_space_tag = - rman_get_bustag(adapter->memory); + adapter->osdep.mem_bus_space_tag = rman_get_bustag(adapter->memory); adapter->osdep.mem_bus_space_handle = rman_get_bushandle(adapter->memory); adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; - adapter->hw.back = &adapter->osdep; - - return (0); -} - -/********************************************************************* - * - * Setup the Legacy or MSI Interrupt handler - * - **********************************************************************/ -int -em_allocate_legacy(struct adapter *adapter) -{ - device_t dev = adapter->dev; - struct tx_ring *txr = adapter->tx_rings; - int error, rid = 0; - - /* Manually turn off all interrupts */ - E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); - - if (adapter->msix == 1) /* using MSI */ - rid = 1; - /* We allocate a single interrupt resource */ - adapter->res = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); - if (adapter->res == NULL) { - device_printf(dev, "Unable to allocate bus resource: " - "interrupt\n"); - return (ENXIO); + /* Only older adapters use IO mapping */ + if (adapter->hw.mac.type < em_mac_min && + adapter->hw.mac.type > e1000_82543) { + /* Figure our where our IO BAR is ? */ + for (rid = PCIR_BAR(0); rid < PCIR_CIS;) { + val = pci_read_config(dev, rid, 4); + if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) { + adapter->io_rid = rid; + break; + } + rid += 4; + /* check for 64bit BAR */ + if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT) + rid += 4; + } + if (rid >= PCIR_CIS) { + device_printf(dev, "Unable to locate IO BAR\n"); + return (ENXIO); + } + adapter->ioport = bus_alloc_resource_any(dev, + SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE); + if (adapter->ioport == NULL) { + device_printf(dev, "Unable to allocate bus resource: " + "ioport\n"); + return (ENXIO); + } + adapter->hw.io_base = 0; + adapter->osdep.io_bus_space_tag = + rman_get_bustag(adapter->ioport); + adapter->osdep.io_bus_space_handle = + rman_get_bushandle(adapter->ioport); } - /* - * Allocate a fast interrupt and the associated - * deferred processing contexts. - */ - TASK_INIT(&adapter->que_task, 0, em_handle_que, adapter); - adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT, - taskqueue_thread_enqueue, &adapter->tq); - taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s que", - device_get_nameunit(adapter->dev)); - /* Use a TX only tasklet for local timer */ - TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr); - txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT, - taskqueue_thread_enqueue, &txr->tq); - taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq", - device_get_nameunit(adapter->dev)); - TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter); - if ((error = bus_setup_intr(dev, adapter->res, INTR_TYPE_NET, - em_irq_fast, NULL, adapter, &adapter->tag)) != 0) { - device_printf(dev, "Failed to register fast interrupt " - "handler: %d\n", error); - taskqueue_free(adapter->tq); - adapter->tq = NULL; - return (error); - } - + adapter->hw.back = &adapter->osdep; + return (0); } /********************************************************************* * * Setup the MSIX Interrupt handlers - * This is not really Multiqueue, rather - * its just separate interrupt vectors - * for TX, RX, and Link. * **********************************************************************/ -int -em_allocate_msix(struct adapter *adapter) +static int +em_if_msix_intr_assign(if_ctx_t ctx, int msix) { - device_t dev = adapter->dev; - struct tx_ring *txr = adapter->tx_rings; - struct rx_ring *rxr = adapter->rx_rings; - int error, rid, vector = 0; - int cpu_id = 0; - - - /* Make sure all interrupts are disabled */ - E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); + struct adapter *adapter = iflib_get_softc(ctx); + struct em_rx_queue *rx_que = adapter->rx_queues; + struct em_tx_queue *tx_que = adapter->tx_queues; + int error, rid, i, vector = 0, rx_vectors; + char buf[16]; /* First set up ring resources */ - for (int i = 0; i < adapter->num_queues; i++, rxr++, vector++) { - - /* RX ring */ + for (i = 0; i < adapter->rx_num_queues; i++, rx_que++, vector++) { rid = vector + 1; - - rxr->res = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &rid, RF_ACTIVE); - if (rxr->res == NULL) { - device_printf(dev, - "Unable to allocate bus resource: " - "RX MSIX Interrupt %d\n", i); - return (ENXIO); - } - if ((error = bus_setup_intr(dev, rxr->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_rx, - rxr, &rxr->tag)) != 0) { - device_printf(dev, "Failed to register RX handler"); - return (error); + snprintf(buf, sizeof(buf), "rxq%d", i); + error = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid, IFLIB_INTR_RXTX, em_msix_que, rx_que, rx_que->me, buf); + if (error) { + device_printf(iflib_get_dev(ctx), "Failed to allocate que int %d err: %d", i, error); + adapter->rx_num_queues = i + 1; + goto fail; } -#if __FreeBSD_version >= 800504 - bus_describe_intr(dev, rxr->res, rxr->tag, "rx%d", i); -#endif - rxr->msix = vector; - - if (em_last_bind_cpu < 0) - em_last_bind_cpu = CPU_FIRST(); - cpu_id = em_last_bind_cpu; - bus_bind_intr(dev, rxr->res, cpu_id); - - TASK_INIT(&rxr->rx_task, 0, em_handle_rx, rxr); - rxr->tq = taskqueue_create_fast("em_rxq", M_NOWAIT, - taskqueue_thread_enqueue, &rxr->tq); - taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq (cpuid %d)", - device_get_nameunit(adapter->dev), cpu_id); - /* - ** Set the bit to enable interrupt - ** in E1000_IMS -- bits 20 and 21 - ** are for RX0 and RX1, note this has - ** NOTHING to do with the MSIX vector - */ - rxr->ims = 1 << (20 + i); - adapter->ims |= rxr->ims; - adapter->ivars |= (8 | rxr->msix) << (i * 4); - em_last_bind_cpu = CPU_NEXT(em_last_bind_cpu); + rx_que->msix = vector; + + /* + * Set the bit to enable interrupt + * in E1000_IMS -- bits 20 and 21 + * are for RX0 and RX1, note this has + * NOTHING to do with the MSIX vector + */ + if (adapter->hw.mac.type == e1000_82574) { + rx_que->eims = 1 << (20 + i); + adapter->ims |= rx_que->eims; + adapter->ivars |= (8 | rx_que->msix) << (i * 4); + } else if (adapter->hw.mac.type == e1000_82575) + rx_que->eims = E1000_EICR_TX_QUEUE0 << vector; + else + rx_que->eims = 1 << vector; } + rx_vectors = vector; - for (int i = 0; i < adapter->num_queues; i++, txr++, vector++) { - /* TX ring */ + vector = 0; + for (i = 0; i < adapter->tx_num_queues; i++, tx_que++, vector++) { rid = vector + 1; - txr->res = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &rid, RF_ACTIVE); - if (txr->res == NULL) { - device_printf(dev, - "Unable to allocate bus resource: " - "TX MSIX Interrupt %d\n", i); - return (ENXIO); - } - if ((error = bus_setup_intr(dev, txr->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx, - txr, &txr->tag)) != 0) { - device_printf(dev, "Failed to register TX handler"); - return (error); - } -#if __FreeBSD_version >= 800504 - bus_describe_intr(dev, txr->res, txr->tag, "tx%d", i); -#endif - txr->msix = vector; - - if (em_last_bind_cpu < 0) - em_last_bind_cpu = CPU_FIRST(); - cpu_id = em_last_bind_cpu; - bus_bind_intr(dev, txr->res, cpu_id); - - TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr); - txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT, - taskqueue_thread_enqueue, &txr->tq); - taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq (cpuid %d)", - device_get_nameunit(adapter->dev), cpu_id); - /* - ** Set the bit to enable interrupt - ** in E1000_IMS -- bits 22 and 23 - ** are for TX0 and TX1, note this has - ** NOTHING to do with the MSIX vector - */ - txr->ims = 1 << (22 + i); - adapter->ims |= txr->ims; - adapter->ivars |= (8 | txr->msix) << (8 + (i * 4)); + snprintf(buf, sizeof(buf), "txq%d", i); + tx_que = &adapter->tx_queues[i]; + iflib_softirq_alloc_generic(ctx, rid, IFLIB_INTR_TX, tx_que, tx_que->me, buf); + + tx_que->msix = (vector % adapter->tx_num_queues); - em_last_bind_cpu = CPU_NEXT(em_last_bind_cpu); + /* + * Set the bit to enable interrupt + * in E1000_IMS -- bits 22 and 23 + * are for TX0 and TX1, note this has + * NOTHING to do with the MSIX vector + */ + if (adapter->hw.mac.type == e1000_82574) { + tx_que->eims = 1 << (22 + i); + adapter->ims |= tx_que->eims; + adapter->ivars |= (8 | tx_que->msix) << (8 + (i * 4)); + } else if (adapter->hw.mac.type == e1000_82575) { + tx_que->eims = E1000_EICR_TX_QUEUE0 << (i % adapter->tx_num_queues); + } else { + tx_que->eims = 1 << (i % adapter->tx_num_queues); + } } /* Link interrupt */ - rid = vector + 1; - adapter->res = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); - if (!adapter->res) { - device_printf(dev,"Unable to allocate " - "bus resource: Link interrupt [%d]\n", rid); - return (ENXIO); - } - /* Set the link handler function */ - error = bus_setup_intr(dev, adapter->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, - em_msix_link, adapter, &adapter->tag); + rid = rx_vectors + 1; + error = iflib_irq_alloc_generic(ctx, &adapter->irq, rid, IFLIB_INTR_ADMIN, em_msix_link, adapter, 0, "aq"); + if (error) { - adapter->res = NULL; - device_printf(dev, "Failed to register LINK handler"); - return (error); + device_printf(iflib_get_dev(ctx), "Failed to register admin handler"); + goto fail; + } + adapter->linkvec = rx_vectors; + if (adapter->hw.mac.type < igb_mac_min) { + adapter->ivars |= (8 | rx_vectors) << 16; + adapter->ivars |= 0x80000000; } -#if __FreeBSD_version >= 800504 - bus_describe_intr(dev, adapter->res, adapter->tag, "link"); -#endif - adapter->linkvec = vector; - adapter->ivars |= (8 | vector) << 16; - adapter->ivars |= 0x80000000; - return (0); +fail: + iflib_irq_free(ctx, &adapter->irq); + rx_que = adapter->rx_queues; + for (int i = 0; i < adapter->rx_num_queues; i++, rx_que++) + iflib_irq_free(ctx, &rx_que->que_irq); + return (error); } - static void -em_free_pci_resources(struct adapter *adapter) +igb_configure_queues(struct adapter *adapter) { - device_t dev = adapter->dev; - struct tx_ring *txr; - struct rx_ring *rxr; - int rid; + struct e1000_hw *hw = &adapter->hw; + struct em_rx_queue *rx_que; + struct em_tx_queue *tx_que; + u32 tmp, ivar = 0, newitr = 0; + /* First turn on RSS capability */ + if (adapter->hw.mac.type != e1000_82575) + E1000_WRITE_REG(hw, E1000_GPIE, + E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME | + E1000_GPIE_PBA | E1000_GPIE_NSICR); - /* - ** Release all the queue interrupt resources: - */ - for (int i = 0; i < adapter->num_queues; i++) { - txr = &adapter->tx_rings[i]; - /* an early abort? */ - if (txr == NULL) - break; - rid = txr->msix +1; - if (txr->tag != NULL) { - bus_teardown_intr(dev, txr->res, txr->tag); - txr->tag = NULL; + /* Turn on MSIX */ + switch (adapter->hw.mac.type) { + case e1000_82580: + case e1000_i350: + case e1000_i354: + case e1000_i210: + case e1000_i211: + case e1000_vfadapt: + case e1000_vfadapt_i350: + /* RX entries */ + for (int i = 0; i < adapter->rx_num_queues; i++) { + u32 index = i >> 1; + ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); + rx_que = &adapter->rx_queues[i]; + if (i & 1) { + ivar &= 0xFF00FFFF; + ivar |= (rx_que->msix | E1000_IVAR_VALID) << 16; + } else { + ivar &= 0xFFFFFF00; + ivar |= rx_que->msix | E1000_IVAR_VALID; + } + E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); + } + /* TX entries */ + for (int i = 0; i < adapter->tx_num_queues; i++) { + u32 index = i >> 1; + ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); + tx_que = &adapter->tx_queues[i]; + if (i & 1) { + ivar &= 0x00FFFFFF; + ivar |= (tx_que->msix | E1000_IVAR_VALID) << 24; + } else { + ivar &= 0xFFFF00FF; + ivar |= (tx_que->msix | E1000_IVAR_VALID) << 8; + } + E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); + adapter->que_mask |= tx_que->eims; } - if (txr->res != NULL) - bus_release_resource(dev, SYS_RES_IRQ, - rid, txr->res); - rxr = &adapter->rx_rings[i]; - /* an early abort? */ - if (rxr == NULL) - break; - rid = rxr->msix +1; - if (rxr->tag != NULL) { - bus_teardown_intr(dev, rxr->res, rxr->tag); - rxr->tag = NULL; + /* And for the link interrupt */ + ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8; + adapter->link_mask = 1 << adapter->linkvec; + E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); + break; + case e1000_82576: + /* RX entries */ + for (int i = 0; i < adapter->rx_num_queues; i++) { + u32 index = i & 0x7; /* Each IVAR has two entries */ + ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); + rx_que = &adapter->rx_queues[i]; + if (i < 8) { + ivar &= 0xFFFFFF00; + ivar |= rx_que->msix | E1000_IVAR_VALID; + } else { + ivar &= 0xFF00FFFF; + ivar |= (rx_que->msix | E1000_IVAR_VALID) << 16; + } + E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); + adapter->que_mask |= rx_que->eims; + } + /* TX entries */ + for (int i = 0; i < adapter->tx_num_queues; i++) { + u32 index = i & 0x7; /* Each IVAR has two entries */ + ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); + tx_que = &adapter->tx_queues[i]; + if (i < 8) { + ivar &= 0xFFFF00FF; + ivar |= (tx_que->msix | E1000_IVAR_VALID) << 8; + } else { + ivar &= 0x00FFFFFF; + ivar |= (tx_que->msix | E1000_IVAR_VALID) << 24; + } + E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); + adapter->que_mask |= tx_que->eims; } - if (rxr->res != NULL) - bus_release_resource(dev, SYS_RES_IRQ, - rid, rxr->res); + + /* And for the link interrupt */ + ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8; + adapter->link_mask = 1 << adapter->linkvec; + E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); + break; + + case e1000_82575: + /* enable MSI-X support*/ + tmp = E1000_READ_REG(hw, E1000_CTRL_EXT); + tmp |= E1000_CTRL_EXT_PBA_CLR; + /* Auto-Mask interrupts upon ICR read. */ + tmp |= E1000_CTRL_EXT_EIAME; + tmp |= E1000_CTRL_EXT_IRCA; + E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp); + + /* Queues */ + for (int i = 0; i < adapter->rx_num_queues; i++) { + rx_que = &adapter->rx_queues[i]; + tmp = E1000_EICR_RX_QUEUE0 << i; + tmp |= E1000_EICR_TX_QUEUE0 << i; + rx_que->eims = tmp; + E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), + i, rx_que->eims); + adapter->que_mask |= rx_que->eims; + } + + /* Link */ + E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec), + E1000_EIMS_OTHER); + adapter->link_mask |= E1000_EIMS_OTHER; + default: + break; } - if (adapter->linkvec) /* we are doing MSIX */ - rid = adapter->linkvec + 1; - else - (adapter->msix != 0) ? (rid = 1):(rid = 0); + /* Set the starting interrupt rate */ + if (em_max_interrupt_rate > 0) + newitr = (4000000 / em_max_interrupt_rate) & 0x7FFC; + + if (hw->mac.type == e1000_82575) + newitr |= newitr << 16; + else + newitr |= E1000_EITR_CNT_IGNR; - if (adapter->tag != NULL) { - bus_teardown_intr(dev, adapter->res, adapter->tag); - adapter->tag = NULL; + for (int i = 0; i < adapter->rx_num_queues; i++) { + rx_que = &adapter->rx_queues[i]; + E1000_WRITE_REG(hw, E1000_EITR(rx_que->msix), newitr); } - if (adapter->res != NULL) - bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res); + return; +} +static void +em_free_pci_resources(if_ctx_t ctx) +{ + struct adapter *adapter = iflib_get_softc(ctx); + struct em_rx_queue *que = adapter->rx_queues; + device_t dev = iflib_get_dev(ctx); - if (adapter->msix) - pci_release_msi(dev); + /* Release all msix queue resources */ + if (adapter->intr_type == IFLIB_INTR_MSIX) + iflib_irq_free(ctx, &adapter->irq); - if (adapter->msix_mem != NULL) - bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem); + for (int i = 0; i < adapter->rx_num_queues; i++, que++) { + iflib_irq_free(ctx, &que->que_irq); + } - if (adapter->memory != NULL) + /* First release all the interrupt resources */ + if (adapter->memory != NULL) { bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(0), adapter->memory); + PCIR_BAR(0), adapter->memory); + adapter->memory = NULL; + } - if (adapter->flash != NULL) + if (adapter->flash != NULL) { bus_release_resource(dev, SYS_RES_MEMORY, - EM_FLASH, adapter->flash); + EM_FLASH, adapter->flash); + adapter->flash = NULL; + } + if (adapter->ioport != NULL) + bus_release_resource(dev, SYS_RES_IOPORT, + adapter->io_rid, adapter->ioport); } -/* - * Setup MSI or MSI/X - */ +/* Setup MSI or MSI/X */ static int -em_setup_msix(struct adapter *adapter) +em_setup_msix(if_ctx_t ctx) { - device_t dev = adapter->dev; - int val; - - /* Nearly always going to use one queue */ - adapter->num_queues = 1; - - /* - ** Try using MSI-X for Hartwell adapters - */ - if ((adapter->hw.mac.type == e1000_82574) && - (em_enable_msix == TRUE)) { -#ifdef EM_MULTIQUEUE - adapter->num_queues = (em_num_queues == 1) ? 1 : 2; - if (adapter->num_queues > 1) - em_enable_vectors_82574(adapter); -#endif - /* Map the MSIX BAR */ - int rid = PCIR_BAR(EM_MSIX_BAR); - adapter->msix_mem = bus_alloc_resource_any(dev, - SYS_RES_MEMORY, &rid, RF_ACTIVE); - if (adapter->msix_mem == NULL) { - /* May not be enabled */ - device_printf(adapter->dev, - "Unable to map MSIX table \n"); - goto msi; - } - val = pci_msix_count(dev); - -#ifdef EM_MULTIQUEUE - /* We need 5 vectors in the multiqueue case */ - if (adapter->num_queues > 1 ) { - if (val >= 5) - val = 5; - else { - adapter->num_queues = 1; - device_printf(adapter->dev, - "Insufficient MSIX vectors for >1 queue, " - "using single queue...\n"); - goto msix_one; - } - } else { -msix_one: -#endif - if (val >= 3) - val = 3; - else { - device_printf(adapter->dev, - "Insufficient MSIX vectors, using MSI\n"); - goto msi; - } -#ifdef EM_MULTIQUEUE - } -#endif + struct adapter *adapter = iflib_get_softc(ctx); - if ((pci_alloc_msix(dev, &val) == 0)) { - device_printf(adapter->dev, - "Using MSIX interrupts " - "with %d vectors\n", val); - return (val); - } - - /* - ** If MSIX alloc failed or provided us with - ** less than needed, free and fall through to MSI - */ - pci_release_msi(dev); - } -msi: - if (adapter->msix_mem != NULL) { - bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem); - adapter->msix_mem = NULL; + if (adapter->hw.mac.type == e1000_82574) { + em_enable_vectors_82574(ctx); } - val = 1; - if (pci_alloc_msi(dev, &val) == 0) { - device_printf(adapter->dev, "Using an MSI interrupt\n"); - return (val); - } - /* Should only happen due to manual configuration */ - device_printf(adapter->dev,"No MSI/MSIX using a Legacy IRQ\n"); return (0); } +/********************************************************************* + * + * Initialize the hardware to a configuration + * as specified by the adapter structure. + * + **********************************************************************/ -/* -** The 3 following flush routines are used as a workaround in the -** I219 client parts and only for them. -** -** em_flush_tx_ring - remove all descriptors from the tx_ring -** -** We want to clear all pending descriptors from the TX ring. -** zeroing happens when the HW reads the regs. We assign the ring itself as -** the data of the next descriptor. We don't care about the data we are about -** to reset the HW. -*/ -static void -em_flush_tx_ring(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - struct tx_ring *txr = adapter->tx_rings; - struct e1000_tx_desc *txd; - u32 tctl, txd_lower = E1000_TXD_CMD_IFCS; - u16 size = 512; - - tctl = E1000_READ_REG(hw, E1000_TCTL); - E1000_WRITE_REG(hw, E1000_TCTL, tctl | E1000_TCTL_EN); - - txd = &txr->tx_base[txr->next_avail_desc++]; - if (txr->next_avail_desc == adapter->num_tx_desc) - txr->next_avail_desc = 0; - - /* Just use the ring as a dummy buffer addr */ - txd->buffer_addr = txr->txdma.dma_paddr; - txd->lower.data = htole32(txd_lower | size); - txd->upper.data = 0; - - /* flush descriptors to memory before notifying the HW */ - wmb(); - - E1000_WRITE_REG(hw, E1000_TDT(0), txr->next_avail_desc); - mb(); - usec_delay(250); -} - -/* -** em_flush_rx_ring - remove all descriptors from the rx_ring -** -** Mark all descriptors in the RX ring as consumed and disable the rx ring -*/ static void -em_flush_rx_ring(struct adapter *adapter) +lem_smartspeed(struct adapter *adapter) { - struct e1000_hw *hw = &adapter->hw; - u32 rctl, rxdctl; - - rctl = E1000_READ_REG(hw, E1000_RCTL); - E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); - E1000_WRITE_FLUSH(hw); - usec_delay(150); - - rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0)); - /* zero the lower 14 bits (prefetch and host thresholds) */ - rxdctl &= 0xffffc000; - /* - * update thresholds: prefetch threshold to 31, host threshold to 1 - * and make sure the granularity is "descriptors" and not "cache lines" - */ - rxdctl |= (0x1F | (1 << 8) | E1000_RXDCTL_THRESH_UNIT_DESC); - E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl); - - /* momentarily enable the RX ring for the changes to take effect */ - E1000_WRITE_REG(hw, E1000_RCTL, rctl | E1000_RCTL_EN); - E1000_WRITE_FLUSH(hw); - usec_delay(150); - E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); -} + u16 phy_tmp; -/* -** em_flush_desc_rings - remove all descriptors from the descriptor rings -** -** In i219, the descriptor rings must be emptied before resetting the HW -** or before changing the device state to D3 during runtime (runtime PM). -** -** Failure to do this will cause the HW to enter a unit hang state which can -** only be released by PCI reset on the device -** -*/ -static void -em_flush_desc_rings(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - device_t dev = adapter->dev; - u16 hang_state; - u32 fext_nvm11, tdlen; - - /* First, disable MULR fix in FEXTNVM11 */ - fext_nvm11 = E1000_READ_REG(hw, E1000_FEXTNVM11); - fext_nvm11 |= E1000_FEXTNVM11_DISABLE_MULR_FIX; - E1000_WRITE_REG(hw, E1000_FEXTNVM11, fext_nvm11); - - /* do nothing if we're not in faulty state, or if the queue is empty */ - tdlen = E1000_READ_REG(hw, E1000_TDLEN(0)); - hang_state = pci_read_config(dev, PCICFG_DESC_RING_STATUS, 2); - if (!(hang_state & FLUSH_DESC_REQUIRED) || !tdlen) + if (adapter->link_active || (adapter->hw.phy.type != e1000_phy_igp) || + adapter->hw.mac.autoneg == 0 || + (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0) return; - em_flush_tx_ring(adapter); - /* recheck, maybe the fault is caused by the rx ring */ - hang_state = pci_read_config(dev, PCICFG_DESC_RING_STATUS, 2); - if (hang_state & FLUSH_DESC_REQUIRED) - em_flush_rx_ring(adapter); + if (adapter->smartspeed == 0) { + /* If Master/Slave config fault is asserted twice, + * we assume back-to-back */ + e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp); + if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT)) + return; + e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp); + if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) { + e1000_read_phy_reg(&adapter->hw, + PHY_1000T_CTRL, &phy_tmp); + if(phy_tmp & CR_1000T_MS_ENABLE) { + phy_tmp &= ~CR_1000T_MS_ENABLE; + e1000_write_phy_reg(&adapter->hw, + PHY_1000T_CTRL, phy_tmp); + adapter->smartspeed++; + if(adapter->hw.mac.autoneg && + !e1000_copper_link_autoneg(&adapter->hw) && + !e1000_read_phy_reg(&adapter->hw, + PHY_CONTROL, &phy_tmp)) { + phy_tmp |= (MII_CR_AUTO_NEG_EN | + MII_CR_RESTART_AUTO_NEG); + e1000_write_phy_reg(&adapter->hw, + PHY_CONTROL, phy_tmp); + } + } + } + return; + } else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) { + /* If still no link, perhaps using 2/3 pair cable */ + e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp); + phy_tmp |= CR_1000T_MS_ENABLE; + e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp); + if(adapter->hw.mac.autoneg && + !e1000_copper_link_autoneg(&adapter->hw) && + !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) { + phy_tmp |= (MII_CR_AUTO_NEG_EN | + MII_CR_RESTART_AUTO_NEG); + e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp); + } + } + /* Restart process after EM_SMARTSPEED_MAX iterations */ + if(adapter->smartspeed++ == EM_SMARTSPEED_MAX) + adapter->smartspeed = 0; } -/********************************************************************* - * - * Initialize the hardware to a configuration - * as specified by the adapter structure. - * - **********************************************************************/ static void -em_reset(struct adapter *adapter) +em_reset(if_ctx_t ctx) { - device_t dev = adapter->dev; - if_t ifp = adapter->ifp; - struct e1000_hw *hw = &adapter->hw; - u16 rx_buffer_size; - u32 pba; + device_t dev = iflib_get_dev(ctx); + struct adapter *adapter = iflib_get_softc(ctx); + struct ifnet *ifp = iflib_get_ifp(ctx); + struct e1000_hw *hw = &adapter->hw; + u16 rx_buffer_size; + u32 pba; INIT_DEBUGOUT("em_reset: begin"); @@ -3077,13 +2272,62 @@ em_reset(struct adapter *adapter) case e1000_pch_spt: pba = E1000_PBA_26K; break; + case e1000_82575: + pba = E1000_PBA_32K; + break; + case e1000_82576: + case e1000_vfadapt: + pba = E1000_READ_REG(hw, E1000_RXPBS); + pba &= E1000_RXPBS_SIZE_MASK_82576; + break; + case e1000_82580: + case e1000_i350: + case e1000_i354: + case e1000_vfadapt_i350: + pba = E1000_READ_REG(hw, E1000_RXPBS); + pba = e1000_rxpbs_adjust_82580(pba); + break; + case e1000_i210: + case e1000_i211: + pba = E1000_PBA_34K; + break; default: if (adapter->hw.mac.max_frame_size > 8192) pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */ else pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */ } - E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba); + + /* Special needs in case of Jumbo frames */ + if ((hw->mac.type == e1000_82575) && (ifp->if_mtu > ETHERMTU)) { + u32 tx_space, min_tx, min_rx; + pba = E1000_READ_REG(hw, E1000_PBA); + tx_space = pba >> 16; + pba &= 0xffff; + min_tx = (adapter->hw.mac.max_frame_size + + sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2; + min_tx = roundup2(min_tx, 1024); + min_tx >>= 10; + min_rx = adapter->hw.mac.max_frame_size; + min_rx = roundup2(min_rx, 1024); + min_rx >>= 10; + if (tx_space < min_tx && + ((min_tx - tx_space) < pba)) { + pba = pba - (min_tx - tx_space); + /* + * if short on rx space, rx wins + * and must trump tx adjustment + */ + if (pba < min_rx) + pba = min_rx; + } + E1000_WRITE_REG(hw, E1000_PBA, pba); + } + + if (hw->mac.type < igb_mac_min) + E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba); + + INIT_DEBUGOUT1("em_reset: pba=%dK",pba); /* * These parameters control the automatic generation (Tx) and @@ -3099,7 +2343,7 @@ em_reset(struct adapter *adapter) * by 1500. * - The pause time is fairly large at 1000 x 512ns = 512 usec. */ - rx_buffer_size = ((E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10 ); + rx_buffer_size = (pba & 0xffff) << 10; hw->fc.high_water = rx_buffer_size - roundup2(adapter->hw.mac.max_frame_size, 1024); hw->fc.low_water = hw->fc.high_water - 1500; @@ -3120,7 +2364,7 @@ em_reset(struct adapter *adapter) switch (hw->mac.type) { case e1000_pchlan: /* Workaround: no TX flow ctrl for PCH */ - hw->fc.requested_mode = e1000_fc_rx_pause; + hw->fc.requested_mode = e1000_fc_rx_pause; hw->fc.pause_time = 0xFFFF; /* override */ if (if_getmtu(ifp) > ETHERMTU) { hw->fc.high_water = 0x3500; @@ -3144,13 +2388,28 @@ em_reset(struct adapter *adapter) else E1000_WRITE_REG(hw, E1000_PBA, 26); break; + case e1000_82575: + case e1000_82576: + /* 8-byte granularity */ + hw->fc.low_water = hw->fc.high_water - 8; + break; + case e1000_82580: + case e1000_i350: + case e1000_i354: + case e1000_i210: + case e1000_i211: + case e1000_vfadapt: + case e1000_vfadapt_i350: + /* 16-byte granularity */ + hw->fc.low_water = hw->fc.high_water - 16; + break; case e1000_ich9lan: case e1000_ich10lan: if (if_getmtu(ifp) > ETHERMTU) { hw->fc.high_water = 0x2800; hw->fc.low_water = hw->fc.high_water - 8; break; - } + } /* else fall thru */ default: if (hw->mac.type == e1000_80003es2lan) @@ -3158,13 +2417,9 @@ em_reset(struct adapter *adapter) break; } - /* I219 needs some special flushing to avoid hangs */ - if (hw->mac.type == e1000_pch_spt) - em_flush_desc_rings(adapter); - /* Issue a global reset */ e1000_reset_hw(hw); - E1000_WRITE_REG(hw, E1000_WUC, 0); + E1000_WRITE_REG(hw, E1000_WUFC, 0); em_disable_aspm(adapter); /* and a re-init */ if (e1000_init_hw(hw) < 0) { @@ -3175,7 +2430,145 @@ em_reset(struct adapter *adapter) E1000_WRITE_REG(hw, E1000_VET, ETHERTYPE_VLAN); e1000_get_phy_info(hw); e1000_check_for_link(hw); - return; +} + +#define RSSKEYLEN 10 +static void +em_initialize_rss_mapping(struct adapter *adapter) +{ + uint8_t rss_key[4 * RSSKEYLEN]; + uint32_t reta = 0; + struct e1000_hw *hw = &adapter->hw; + int i; + + /* + * Configure RSS key + */ + arc4rand(rss_key, sizeof(rss_key), 0); + for (i = 0; i < RSSKEYLEN; ++i) { + uint32_t rssrk = 0; + + rssrk = EM_RSSRK_VAL(rss_key, i); + E1000_WRITE_REG(hw,E1000_RSSRK(i), rssrk); + } + + /* + * Configure RSS redirect table in following fashion: + * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)] + */ + for (i = 0; i < sizeof(reta); ++i) { + uint32_t q; + + q = (i % adapter->rx_num_queues) << 7; + reta |= q << (8 * i); + } + + for (i = 0; i < 32; ++i) + E1000_WRITE_REG(hw, E1000_RETA(i), reta); + + E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_RSS_ENABLE_2Q | + E1000_MRQC_RSS_FIELD_IPV4_TCP | + E1000_MRQC_RSS_FIELD_IPV4 | + E1000_MRQC_RSS_FIELD_IPV6_TCP_EX | + E1000_MRQC_RSS_FIELD_IPV6_EX | + E1000_MRQC_RSS_FIELD_IPV6); + +} + +static void +igb_initialize_rss_mapping(struct adapter *adapter) +{ + struct e1000_hw *hw = &adapter->hw; + int i; + int queue_id; + u32 reta; + u32 rss_key[10], mrqc, shift = 0; + + /* XXX? */ + if (adapter->hw.mac.type == e1000_82575) + shift = 6; + + /* + * The redirection table controls which destination + * queue each bucket redirects traffic to. + * Each DWORD represents four queues, with the LSB + * being the first queue in the DWORD. + * + * This just allocates buckets to queues using round-robin + * allocation. + * + * NOTE: It Just Happens to line up with the default + * RSS allocation method. + */ + + /* Warning FM follows */ + reta = 0; + for (i = 0; i < 128; i++) { +#ifdef RSS + queue_id = rss_get_indirection_to_bucket(i); + /* + * If we have more queues than buckets, we'll + * end up mapping buckets to a subset of the + * queues. + * + * If we have more buckets than queues, we'll + * end up instead assigning multiple buckets + * to queues. + * + * Both are suboptimal, but we need to handle + * the case so we don't go out of bounds + * indexing arrays and such. + */ + queue_id = queue_id % adapter->rx_num_queues; +#else + queue_id = (i % adapter->rx_num_queues); +#endif + /* Adjust if required */ + queue_id = queue_id << shift; + + /* + * The low 8 bits are for hash value (n+0); + * The next 8 bits are for hash value (n+1), etc. + */ + reta = reta >> 8; + reta = reta | ( ((uint32_t) queue_id) << 24); + if ((i & 3) == 3) { + E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta); + reta = 0; + } + } + + /* Now fill in hash table */ + + /* + * MRQC: Multiple Receive Queues Command + * Set queuing to RSS control, number depends on the device. + */ + mrqc = E1000_MRQC_ENABLE_RSS_8Q; + +#ifdef RSS + /* XXX ew typecasting */ + rss_getkey((uint8_t *) &rss_key); +#else + arc4rand(&rss_key, sizeof(rss_key), 0); +#endif + for (i = 0; i < 10; i++) + E1000_WRITE_REG_ARRAY(hw, + E1000_RSSRK(0), i, rss_key[i]); + + /* + * Configure the RSS fields to hash upon. + */ + mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | + E1000_MRQC_RSS_FIELD_IPV4_TCP); + mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 | + E1000_MRQC_RSS_FIELD_IPV6_TCP); + mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP | + E1000_MRQC_RSS_FIELD_IPV6_UDP); + mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | + E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); + + E1000_WRITE_REG(hw, E1000_MRQC, mrqc); } /********************************************************************* @@ -3184,497 +2577,239 @@ em_reset(struct adapter *adapter) * **********************************************************************/ static int -em_setup_interface(device_t dev, struct adapter *adapter) +em_setup_interface(if_ctx_t ctx) { - if_t ifp; + struct ifnet *ifp = iflib_get_ifp(ctx); + struct adapter *adapter = iflib_get_softc(ctx); + if_softc_ctx_t scctx = adapter->shared; + uint64_t cap = 0; INIT_DEBUGOUT("em_setup_interface: begin"); - ifp = adapter->ifp = if_gethandle(IFT_ETHER); - if (ifp == 0) { - device_printf(dev, "can not allocate ifnet structure\n"); - return (-1); - } - if_initname(ifp, device_get_name(dev), device_get_unit(dev)); - if_setdev(ifp, dev); - if_setinitfn(ifp, em_init); - if_setsoftc(ifp, adapter); - if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); - if_setioctlfn(ifp, em_ioctl); - if_setgetcounterfn(ifp, em_get_counter); - /* TSO parameters */ - ifp->if_hw_tsomax = IP_MAXPACKET; + if_sethwtsomax(ifp, IP_MAXPACKET); /* Take m_pullup(9)'s in em_xmit() w/ TSO into acount. */ - ifp->if_hw_tsomaxsegcount = EM_MAX_SCATTER - 5; - ifp->if_hw_tsomaxsegsize = EM_TSO_SEG_SIZE; - -#ifdef EM_MULTIQUEUE - /* Multiqueue stack interface */ - if_settransmitfn(ifp, em_mq_start); - if_setqflushfn(ifp, em_qflush); -#else - if_setstartfn(ifp, em_start); - if_setsendqlen(ifp, adapter->num_tx_desc - 1); - if_setsendqready(ifp); -#endif - - ether_ifattach(ifp, adapter->hw.mac.addr); + if_sethwtsomaxsegcount(ifp, EM_MAX_SCATTER - 5); + if_sethwtsomaxsegsize(ifp, EM_TSO_SEG_SIZE); - if_setcapabilities(ifp, 0); - if_setcapenable(ifp, 0); + /* Single Queue */ + if (adapter->tx_num_queues == 1) { + if_setsendqlen(ifp, scctx->isc_ntxd[0] - 1); + if_setsendqready(ifp); + } + cap = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO4; + cap |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWTSO | IFCAP_VLAN_MTU; - if_setcapabilitiesbit(ifp, IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | - IFCAP_TSO4, 0); /* * Tell the upper layer(s) we * support full VLAN capability */ if_setifheaderlen(ifp, sizeof(struct ether_vlan_header)); - if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWTSO | - IFCAP_VLAN_MTU, 0); - if_setcapenable(ifp, if_getcapabilities(ifp)); + if_setcapabilitiesbit(ifp, cap, 0); /* - ** Don't turn this on by default, if vlans are - ** created on another pseudo device (eg. lagg) - ** then vlan events are not passed thru, breaking - ** operation, but with HW FILTER off it works. If - ** using vlans directly on the em driver you can - ** enable this and get full hardware tag filtering. - */ + * Don't turn this on by default, if vlans are + * created on another pseudo device (eg. lagg) + * then vlan events are not passed thru, breaking + * operation, but with HW FILTER off it works. If + * using vlans directly on the em driver you can + * enable this and get full hardware tag filtering. + */ if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWFILTER,0); -#ifdef DEVICE_POLLING - if_setcapabilitiesbit(ifp, IFCAP_POLLING,0); -#endif - /* Enable only WOL MAGIC by default */ if (adapter->wol) { - if_setcapabilitiesbit(ifp, IFCAP_WOL, 0); - if_setcapenablebit(ifp, IFCAP_WOL_MAGIC, 0); + if_setcapenablebit(ifp, IFCAP_WOL_MAGIC, + IFCAP_WOL_MCAST| IFCAP_WOL_UCAST); + } else { + if_setcapenablebit(ifp, 0, IFCAP_WOL_MAGIC | + IFCAP_WOL_MCAST| IFCAP_WOL_UCAST); } - + /* * Specify the media types supported by this adapter and register * callbacks to update media and link information */ - ifmedia_init(&adapter->media, IFM_IMASK, - em_media_change, em_media_status); if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { u_char fiber_type = IFM_1000_SX; /* default type */ - ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX, - 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL); + if (adapter->hw.mac.type == e1000_82545) + fiber_type = IFM_1000_LX; + ifmedia_add(adapter->media, IFM_ETHER | fiber_type | IFM_FDX, 0, NULL); + ifmedia_add(adapter->media, IFM_ETHER | fiber_type, 0, NULL); } else { - ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, - 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, - 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, - 0, NULL); + ifmedia_add(adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); + ifmedia_add(adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); + ifmedia_add(adapter->media, IFM_ETHER | IFM_100_TX, 0, NULL); + ifmedia_add(adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); if (adapter->hw.phy.type != e1000_phy_ife) { - ifmedia_add(&adapter->media, - IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); - ifmedia_add(&adapter->media, - IFM_ETHER | IFM_1000_T, 0, NULL); + ifmedia_add(adapter->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); + ifmedia_add(adapter->media, IFM_ETHER | IFM_1000_T, 0, NULL); } } - ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); - ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); + ifmedia_add(adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); + ifmedia_set(adapter->media, IFM_ETHER | IFM_AUTO); return (0); } - -/* - * Manage DMA'able memory. - */ -static void -em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) -{ - if (error) - return; - *(bus_addr_t *) arg = segs[0].ds_addr; -} - static int -em_dma_malloc(struct adapter *adapter, bus_size_t size, - struct em_dma_alloc *dma, int mapflags) +em_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets) { - int error; - - error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ - EM_DBA_ALIGN, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - size, /* maxsize */ - 1, /* nsegments */ - size, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockarg */ - &dma->dma_tag); - if (error) { - device_printf(adapter->dev, - "%s: bus_dma_tag_create failed: %d\n", - __func__, error); - goto fail_0; - } + struct adapter *adapter = iflib_get_softc(ctx); + if_softc_ctx_t scctx = adapter->shared; + int error = E1000_SUCCESS; + struct em_tx_queue *que; + int i, j; - error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, - BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map); - if (error) { - device_printf(adapter->dev, - "%s: bus_dmamem_alloc(%ju) failed: %d\n", - __func__, (uintmax_t)size, error); - goto fail_2; - } - - dma->dma_paddr = 0; - error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, - size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); - if (error || dma->dma_paddr == 0) { - device_printf(adapter->dev, - "%s: bus_dmamap_load failed: %d\n", - __func__, error); - goto fail_3; - } - - return (0); - -fail_3: - bus_dmamap_unload(dma->dma_tag, dma->dma_map); -fail_2: - bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); - bus_dma_tag_destroy(dma->dma_tag); -fail_0: - dma->dma_tag = NULL; - - return (error); -} - -static void -em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma) -{ - if (dma->dma_tag == NULL) - return; - if (dma->dma_paddr != 0) { - bus_dmamap_sync(dma->dma_tag, dma->dma_map, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(dma->dma_tag, dma->dma_map); - dma->dma_paddr = 0; - } - if (dma->dma_vaddr != NULL) { - bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); - dma->dma_vaddr = NULL; - } - bus_dma_tag_destroy(dma->dma_tag); - dma->dma_tag = NULL; -} - - -/********************************************************************* - * - * Allocate memory for the transmit and receive rings, and then - * the descriptors associated with each, called only once at attach. - * - **********************************************************************/ -static int -em_allocate_queues(struct adapter *adapter) -{ - device_t dev = adapter->dev; - struct tx_ring *txr = NULL; - struct rx_ring *rxr = NULL; - int rsize, tsize, error = E1000_SUCCESS; - int txconf = 0, rxconf = 0; - - - /* Allocate the TX ring struct memory */ - if (!(adapter->tx_rings = - (struct tx_ring *) malloc(sizeof(struct tx_ring) * - adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate TX ring memory\n"); - error = ENOMEM; - goto fail; - } + MPASS(adapter->tx_num_queues > 0); + MPASS(adapter->tx_num_queues == ntxqsets); - /* Now allocate the RX */ - if (!(adapter->rx_rings = - (struct rx_ring *) malloc(sizeof(struct rx_ring) * - adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate RX ring memory\n"); - error = ENOMEM; - goto rx_fail; + /* First allocate the top level queue structs */ + if (!(adapter->tx_queues = + (struct em_tx_queue *) malloc(sizeof(struct em_tx_queue) * + adapter->tx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { + device_printf(iflib_get_dev(ctx), "Unable to allocate queue memory\n"); + return(ENOMEM); } - tsize = roundup2(adapter->num_tx_desc * - sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); - /* - * Now set up the TX queues, txconf is needed to handle the - * possibility that things fail midcourse and we need to - * undo memory gracefully - */ - for (int i = 0; i < adapter->num_queues; i++, txconf++) { + for (i = 0, que = adapter->tx_queues; i < adapter->tx_num_queues; i++, que++) { /* Set up some basics */ - txr = &adapter->tx_rings[i]; - txr->adapter = adapter; - txr->me = i; - - /* Initialize the TX lock */ - snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", - device_get_nameunit(dev), txr->me); - mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); - - if (em_dma_malloc(adapter, tsize, - &txr->txdma, BUS_DMA_NOWAIT)) { - device_printf(dev, - "Unable to allocate TX Descriptor memory\n"); - error = ENOMEM; - goto err_tx_desc; - } - txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr; - bzero((void *)txr->tx_base, tsize); - - if (em_allocate_transmit_buffers(txr)) { - device_printf(dev, - "Critical Failure setting up transmit buffers\n"); - error = ENOMEM; - goto err_tx_desc; - } -#if __FreeBSD_version >= 800000 - /* Allocate a buf ring */ - txr->br = buf_ring_alloc(4096, M_DEVBUF, - M_WAITOK, &txr->tx_mtx); -#endif - } - /* - * Next the RX queues... - */ - rsize = roundup2(adapter->num_rx_desc * - sizeof(union e1000_rx_desc_extended), EM_DBA_ALIGN); - for (int i = 0; i < adapter->num_queues; i++, rxconf++) { - rxr = &adapter->rx_rings[i]; - rxr->adapter = adapter; - rxr->me = i; - - /* Initialize the RX lock */ - snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", - device_get_nameunit(dev), txr->me); - mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); - - if (em_dma_malloc(adapter, rsize, - &rxr->rxdma, BUS_DMA_NOWAIT)) { - device_printf(dev, - "Unable to allocate RxDescriptor memory\n"); - error = ENOMEM; - goto err_rx_desc; - } - rxr->rx_base = (union e1000_rx_desc_extended *)rxr->rxdma.dma_vaddr; - bzero((void *)rxr->rx_base, rsize); + struct tx_ring *txr = &que->txr; + txr->adapter = que->adapter = adapter; + que->me = txr->me = i; - /* Allocate receive buffers for the ring*/ - if (em_allocate_receive_buffers(rxr)) { - device_printf(dev, - "Critical Failure setting up receive buffers\n"); + /* Allocate report status array */ + if (!(txr->tx_rsq = (qidx_t *) malloc(sizeof(qidx_t) * scctx->isc_ntxd[0], M_DEVBUF, M_NOWAIT | M_ZERO))) { + device_printf(iflib_get_dev(ctx), "failed to allocate rs_idxs memory\n"); error = ENOMEM; - goto err_rx_desc; + goto fail; } + for (j = 0; j < scctx->isc_ntxd[0]; j++) + txr->tx_rsq[j] = QIDX_INVALID; + /* get the virtual and physical address of the hardware queues */ + txr->tx_base = (struct e1000_tx_desc *)vaddrs[i*ntxqs]; + txr->tx_paddr = paddrs[i*ntxqs]; } - + + device_printf(iflib_get_dev(ctx), "allocated for %d tx_queues\n", adapter->tx_num_queues); return (0); - -err_rx_desc: - for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) - em_dma_free(adapter, &rxr->rxdma); -err_tx_desc: - for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) - em_dma_free(adapter, &txr->txdma); - free(adapter->rx_rings, M_DEVBUF); -rx_fail: -#if __FreeBSD_version >= 800000 - buf_ring_free(txr->br, M_DEVBUF); -#endif - free(adapter->tx_rings, M_DEVBUF); fail: + em_if_queues_free(ctx); return (error); } - -/********************************************************************* - * - * Allocate memory for tx_buffer structures. The tx_buffer stores all - * the information needed to transmit a packet on the wire. This is - * called only once at attach, setup is done every reset. - * - **********************************************************************/ static int -em_allocate_transmit_buffers(struct tx_ring *txr) +em_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nrxqs, int nrxqsets) { - struct adapter *adapter = txr->adapter; - device_t dev = adapter->dev; - struct em_txbuffer *txbuf; - int error, i; + struct adapter *adapter = iflib_get_softc(ctx); + int error = E1000_SUCCESS; + struct em_rx_queue *que; + int i; - /* - * Setup DMA descriptor areas. - */ - if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - EM_TSO_SIZE, /* maxsize */ - EM_MAX_SCATTER, /* nsegments */ - PAGE_SIZE, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockfuncarg */ - &txr->txtag))) { - device_printf(dev,"Unable to allocate TX DMA tag\n"); - goto fail; - } + MPASS(adapter->rx_num_queues > 0); + MPASS(adapter->rx_num_queues == nrxqsets); - if (!(txr->tx_buffers = - (struct em_txbuffer *) malloc(sizeof(struct em_txbuffer) * - adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate tx_buffer memory\n"); + /* First allocate the top level queue structs */ + if (!(adapter->rx_queues = + (struct em_rx_queue *) malloc(sizeof(struct em_rx_queue) * + adapter->rx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { + device_printf(iflib_get_dev(ctx), "Unable to allocate queue memory\n"); error = ENOMEM; - goto fail; + goto fail; } - /* Create the descriptor buffer dma maps */ - txbuf = txr->tx_buffers; - for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { - error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); - if (error != 0) { - device_printf(dev, "Unable to create TX DMA map\n"); - goto fail; - } + for (i = 0, que = adapter->rx_queues; i < nrxqsets; i++, que++) { + /* Set up some basics */ + struct rx_ring *rxr = &que->rxr; + rxr->adapter = que->adapter = adapter; + rxr->que = que; + que->me = rxr->me = i; + + /* get the virtual and physical address of the hardware queues */ + rxr->rx_base = (union e1000_rx_desc_extended *)vaddrs[i*nrxqs]; + rxr->rx_paddr = paddrs[i*nrxqs]; } - return 0; + device_printf(iflib_get_dev(ctx), "allocated for %d rx_queues\n", adapter->rx_num_queues); + + return (0); fail: - /* We free all, it handles case where we are in the middle */ - em_free_transmit_structures(adapter); + em_if_queues_free(ctx); return (error); } -/********************************************************************* - * - * Initialize a transmit ring. - * - **********************************************************************/ static void -em_setup_transmit_ring(struct tx_ring *txr) +em_if_queues_free(if_ctx_t ctx) { - struct adapter *adapter = txr->adapter; - struct em_txbuffer *txbuf; - int i; -#ifdef DEV_NETMAP - struct netmap_slot *slot; - struct netmap_adapter *na = netmap_getna(adapter->ifp); -#endif /* DEV_NETMAP */ - - /* Clear the old descriptor contents */ - EM_TX_LOCK(txr); -#ifdef DEV_NETMAP - slot = netmap_reset(na, NR_TX, txr->me, 0); -#endif /* DEV_NETMAP */ - - bzero((void *)txr->tx_base, - (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc); - /* Reset indices */ - txr->next_avail_desc = 0; - txr->next_to_clean = 0; - - /* Free any existing tx buffers. */ - txbuf = txr->tx_buffers; - for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { - if (txbuf->m_head != NULL) { - bus_dmamap_sync(txr->txtag, txbuf->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(txr->txtag, txbuf->map); - m_freem(txbuf->m_head); - txbuf->m_head = NULL; - } -#ifdef DEV_NETMAP - if (slot) { - int si = netmap_idx_n2k(&na->tx_rings[txr->me], i); - uint64_t paddr; - void *addr; - - addr = PNMB(na, slot + si, &paddr); - txr->tx_base[i].buffer_addr = htole64(paddr); - /* reload the map for netmap mode */ - netmap_load_map(na, txr->txtag, txbuf->map, addr); - } -#endif /* DEV_NETMAP */ + struct adapter *adapter = iflib_get_softc(ctx); + struct em_tx_queue *tx_que = adapter->tx_queues; + struct em_rx_queue *rx_que = adapter->rx_queues; + + if (tx_que != NULL) { + for (int i = 0; i < adapter->tx_num_queues; i++, tx_que++) { + struct tx_ring *txr = &tx_que->txr; + if (txr->tx_rsq == NULL) + break; - /* clear the watch index */ - txbuf->next_eop = -1; - } + free(txr->tx_rsq, M_DEVBUF); + txr->tx_rsq = NULL; + } + free(adapter->tx_queues, M_DEVBUF); + adapter->tx_queues = NULL; + } - /* Set number of descriptors available */ - txr->tx_avail = adapter->num_tx_desc; - txr->busy = EM_TX_IDLE; + if (rx_que != NULL) { + free(adapter->rx_queues, M_DEVBUF); + adapter->rx_queues = NULL; + } - /* Clear checksum offload context. */ - txr->last_hw_offload = 0; - txr->last_hw_ipcss = 0; - txr->last_hw_ipcso = 0; - txr->last_hw_tucss = 0; - txr->last_hw_tucso = 0; + em_release_hw_control(adapter); - bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - EM_TX_UNLOCK(txr); + if (adapter->mta != NULL) { + free(adapter->mta, M_DEVBUF); + } } /********************************************************************* * - * Initialize all transmit rings. + * Enable transmit unit. * **********************************************************************/ static void -em_setup_transmit_structures(struct adapter *adapter) +em_initialize_transmit_unit(if_ctx_t ctx) { - struct tx_ring *txr = adapter->tx_rings; + struct adapter *adapter = iflib_get_softc(ctx); + if_softc_ctx_t scctx = adapter->shared; + struct em_tx_queue *que; + struct tx_ring *txr; + struct e1000_hw *hw = &adapter->hw; + u32 tctl, txdctl = 0, tarc, tipg = 0; - for (int i = 0; i < adapter->num_queues; i++, txr++) - em_setup_transmit_ring(txr); + INIT_DEBUGOUT("em_initialize_transmit_unit: begin"); - return; -} + for (int i = 0; i < adapter->tx_num_queues; i++, txr++) { + u64 bus_addr; + caddr_t offp, endp; -/********************************************************************* - * - * Enable transmit unit. - * - **********************************************************************/ -static void -em_initialize_transmit_unit(struct adapter *adapter) -{ - struct tx_ring *txr = adapter->tx_rings; - struct e1000_hw *hw = &adapter->hw; - u32 tctl, txdctl = 0, tarc, tipg = 0; + que = &adapter->tx_queues[i]; + txr = &que->txr; + bus_addr = txr->tx_paddr; - INIT_DEBUGOUT("em_initialize_transmit_unit: begin"); + /* Clear checksum offload context. */ + offp = (caddr_t)&txr->csum_flags; + endp = (caddr_t)(txr + 1); + bzero(offp, endp - offp); - for (int i = 0; i < adapter->num_queues; i++, txr++) { - u64 bus_addr = txr->txdma.dma_paddr; /* Base and Len of TX Ring */ E1000_WRITE_REG(hw, E1000_TDLEN(i), - adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); + scctx->isc_ntxd[0] * sizeof(struct e1000_tx_desc)); E1000_WRITE_REG(hw, E1000_TDBAH(i), - (u32)(bus_addr >> 32)); + (u32)(bus_addr >> 32)); E1000_WRITE_REG(hw, E1000_TDBAL(i), - (u32)bus_addr); + (u32)bus_addr); /* Init the HEAD/TAIL indices */ E1000_WRITE_REG(hw, E1000_TDT(i), 0); E1000_WRITE_REG(hw, E1000_TDH(i), 0); @@ -3683,16 +2818,15 @@ em_initialize_transmit_unit(struct adapter *adapter) E1000_READ_REG(&adapter->hw, E1000_TDBAL(i)), E1000_READ_REG(&adapter->hw, E1000_TDLEN(i))); - txr->busy = EM_TX_IDLE; txdctl = 0; /* clear txdctl */ - txdctl |= 0x1f; /* PTHRESH */ - txdctl |= 1 << 8; /* HTHRESH */ - txdctl |= 1 << 16;/* WTHRESH */ + txdctl |= 0x1f; /* PTHRESH */ + txdctl |= 1 << 8; /* HTHRESH */ + txdctl |= 1 << 16;/* WTHRESH */ txdctl |= 1 << 22; /* Reserved bit 22 must always be 1 */ txdctl |= E1000_TXDCTL_GRAN; - txdctl |= 1 << 25; /* LWTHRESH */ + txdctl |= 1 << 25; /* LWTHRESH */ - E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl); + E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl); } /* Set the default values for the Tx Inter Packet Gap timer */ @@ -3702,6 +2836,11 @@ em_initialize_transmit_unit(struct adapter *adapter) tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; break; + case e1000_82542: + tipg = DEFAULT_82542_TIPG_IPGT; + tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; + tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; + break; default: if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || (adapter->hw.phy.media_type == @@ -3736,7 +2875,7 @@ em_initialize_transmit_unit(struct adapter *adapter) } else if (adapter->hw.mac.type == e1000_82574) { tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); tarc |= TARC_ERRATA_BIT; - if ( adapter->num_queues > 1) { + if ( adapter->tx_num_queues > 1) { tarc |= (TARC_COMPENSATION_MODE | TARC_MQ_FIX); E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc); @@ -3744,7 +2883,6 @@ em_initialize_transmit_unit(struct adapter *adapter) E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); } - adapter->txd_cmd = E1000_TXD_CMD_IFCS; if (adapter->tx_int_delay.value > 0) adapter->txd_cmd |= E1000_TXD_CMD_IDE; @@ -3771,780 +2909,6 @@ em_initialize_transmit_unit(struct adapter *adapter) } } - -/********************************************************************* - * - * Free all transmit rings. - * - **********************************************************************/ -static void -em_free_transmit_structures(struct adapter *adapter) -{ - struct tx_ring *txr = adapter->tx_rings; - - for (int i = 0; i < adapter->num_queues; i++, txr++) { - EM_TX_LOCK(txr); - em_free_transmit_buffers(txr); - em_dma_free(adapter, &txr->txdma); - EM_TX_UNLOCK(txr); - EM_TX_LOCK_DESTROY(txr); - } - - free(adapter->tx_rings, M_DEVBUF); -} - -/********************************************************************* - * - * Free transmit ring related data structures. - * - **********************************************************************/ -static void -em_free_transmit_buffers(struct tx_ring *txr) -{ - struct adapter *adapter = txr->adapter; - struct em_txbuffer *txbuf; - - INIT_DEBUGOUT("free_transmit_ring: begin"); - - if (txr->tx_buffers == NULL) - return; - - for (int i = 0; i < adapter->num_tx_desc; i++) { - txbuf = &txr->tx_buffers[i]; - if (txbuf->m_head != NULL) { - bus_dmamap_sync(txr->txtag, txbuf->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(txr->txtag, - txbuf->map); - m_freem(txbuf->m_head); - txbuf->m_head = NULL; - if (txbuf->map != NULL) { - bus_dmamap_destroy(txr->txtag, - txbuf->map); - txbuf->map = NULL; - } - } else if (txbuf->map != NULL) { - bus_dmamap_unload(txr->txtag, - txbuf->map); - bus_dmamap_destroy(txr->txtag, - txbuf->map); - txbuf->map = NULL; - } - } -#if __FreeBSD_version >= 800000 - if (txr->br != NULL) - buf_ring_free(txr->br, M_DEVBUF); -#endif - if (txr->tx_buffers != NULL) { - free(txr->tx_buffers, M_DEVBUF); - txr->tx_buffers = NULL; - } - if (txr->txtag != NULL) { - bus_dma_tag_destroy(txr->txtag); - txr->txtag = NULL; - } - return; -} - - -/********************************************************************* - * The offload context is protocol specific (TCP/UDP) and thus - * only needs to be set when the protocol changes. The occasion - * of a context change can be a performance detriment, and - * might be better just disabled. The reason arises in the way - * in which the controller supports pipelined requests from the - * Tx data DMA. Up to four requests can be pipelined, and they may - * belong to the same packet or to multiple packets. However all - * requests for one packet are issued before a request is issued - * for a subsequent packet and if a request for the next packet - * requires a context change, that request will be stalled - * until the previous request completes. This means setting up - * a new context effectively disables pipelined Tx data DMA which - * in turn greatly slow down performance to send small sized - * frames. - **********************************************************************/ -static void -em_transmit_checksum_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off, - struct ip *ip, u32 *txd_upper, u32 *txd_lower) -{ - struct adapter *adapter = txr->adapter; - struct e1000_context_desc *TXD = NULL; - struct em_txbuffer *tx_buffer; - int cur, hdr_len; - u32 cmd = 0; - u16 offload = 0; - u8 ipcso, ipcss, tucso, tucss; - - ipcss = ipcso = tucss = tucso = 0; - hdr_len = ip_off + (ip->ip_hl << 2); - cur = txr->next_avail_desc; - - /* Setup of IP header checksum. */ - if (mp->m_pkthdr.csum_flags & CSUM_IP) { - *txd_upper |= E1000_TXD_POPTS_IXSM << 8; - offload |= CSUM_IP; - ipcss = ip_off; - ipcso = ip_off + offsetof(struct ip, ip_sum); - /* - * Start offset for header checksum calculation. - * End offset for header checksum calculation. - * Offset of place to put the checksum. - */ - TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; - TXD->lower_setup.ip_fields.ipcss = ipcss; - TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len); - TXD->lower_setup.ip_fields.ipcso = ipcso; - cmd |= E1000_TXD_CMD_IP; - } - - if (mp->m_pkthdr.csum_flags & CSUM_TCP) { - *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; - *txd_upper |= E1000_TXD_POPTS_TXSM << 8; - offload |= CSUM_TCP; - tucss = hdr_len; - tucso = hdr_len + offsetof(struct tcphdr, th_sum); - /* - * The 82574L can only remember the *last* context used - * regardless of queue that it was use for. We cannot reuse - * contexts on this hardware platform and must generate a new - * context every time. 82574L hardware spec, section 7.2.6, - * second note. - */ - if (adapter->num_queues < 2) { - /* - * Setting up new checksum offload context for every - * frames takes a lot of processing time for hardware. - * This also reduces performance a lot for small sized - * frames so avoid it if driver can use previously - * configured checksum offload context. - */ - if (txr->last_hw_offload == offload) { - if (offload & CSUM_IP) { - if (txr->last_hw_ipcss == ipcss && - txr->last_hw_ipcso == ipcso && - txr->last_hw_tucss == tucss && - txr->last_hw_tucso == tucso) - return; - } else { - if (txr->last_hw_tucss == tucss && - txr->last_hw_tucso == tucso) - return; - } - } - txr->last_hw_offload = offload; - txr->last_hw_tucss = tucss; - txr->last_hw_tucso = tucso; - } - /* - * Start offset for payload checksum calculation. - * End offset for payload checksum calculation. - * Offset of place to put the checksum. - */ - TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; - TXD->upper_setup.tcp_fields.tucss = hdr_len; - TXD->upper_setup.tcp_fields.tucse = htole16(0); - TXD->upper_setup.tcp_fields.tucso = tucso; - cmd |= E1000_TXD_CMD_TCP; - } else if (mp->m_pkthdr.csum_flags & CSUM_UDP) { - *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; - *txd_upper |= E1000_TXD_POPTS_TXSM << 8; - tucss = hdr_len; - tucso = hdr_len + offsetof(struct udphdr, uh_sum); - /* - * The 82574L can only remember the *last* context used - * regardless of queue that it was use for. We cannot reuse - * contexts on this hardware platform and must generate a new - * context every time. 82574L hardware spec, section 7.2.6, - * second note. - */ - if (adapter->num_queues < 2) { - /* - * Setting up new checksum offload context for every - * frames takes a lot of processing time for hardware. - * This also reduces performance a lot for small sized - * frames so avoid it if driver can use previously - * configured checksum offload context. - */ - if (txr->last_hw_offload == offload) { - if (offload & CSUM_IP) { - if (txr->last_hw_ipcss == ipcss && - txr->last_hw_ipcso == ipcso && - txr->last_hw_tucss == tucss && - txr->last_hw_tucso == tucso) - return; - } else { - if (txr->last_hw_tucss == tucss && - txr->last_hw_tucso == tucso) - return; - } - } - txr->last_hw_offload = offload; - txr->last_hw_tucss = tucss; - txr->last_hw_tucso = tucso; - } - /* - * Start offset for header checksum calculation. - * End offset for header checksum calculation. - * Offset of place to put the checksum. - */ - TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; - TXD->upper_setup.tcp_fields.tucss = tucss; - TXD->upper_setup.tcp_fields.tucse = htole16(0); - TXD->upper_setup.tcp_fields.tucso = tucso; - } - - if (offload & CSUM_IP) { - txr->last_hw_ipcss = ipcss; - txr->last_hw_ipcso = ipcso; - } - - TXD->tcp_seg_setup.data = htole32(0); - TXD->cmd_and_length = - htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd); - tx_buffer = &txr->tx_buffers[cur]; - tx_buffer->m_head = NULL; - tx_buffer->next_eop = -1; - - if (++cur == adapter->num_tx_desc) - cur = 0; - - txr->tx_avail--; - txr->next_avail_desc = cur; -} - - -/********************************************************************** - * - * Setup work for hardware segmentation offload (TSO) - * - **********************************************************************/ -static void -em_tso_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off, - struct ip *ip, struct tcphdr *tp, u32 *txd_upper, u32 *txd_lower) -{ - struct adapter *adapter = txr->adapter; - struct e1000_context_desc *TXD; - struct em_txbuffer *tx_buffer; - int cur, hdr_len; - - /* - * In theory we can use the same TSO context if and only if - * frame is the same type(IP/TCP) and the same MSS. However - * checking whether a frame has the same IP/TCP structure is - * hard thing so just ignore that and always restablish a - * new TSO context. - */ - hdr_len = ip_off + (ip->ip_hl << 2) + (tp->th_off << 2); - *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ - E1000_TXD_DTYP_D | /* Data descr type */ - E1000_TXD_CMD_TSE); /* Do TSE on this packet */ - - /* IP and/or TCP header checksum calculation and insertion. */ - *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8; - - cur = txr->next_avail_desc; - tx_buffer = &txr->tx_buffers[cur]; - TXD = (struct e1000_context_desc *) &txr->tx_base[cur]; - - /* - * Start offset for header checksum calculation. - * End offset for header checksum calculation. - * Offset of place put the checksum. - */ - TXD->lower_setup.ip_fields.ipcss = ip_off; - TXD->lower_setup.ip_fields.ipcse = - htole16(ip_off + (ip->ip_hl << 2) - 1); - TXD->lower_setup.ip_fields.ipcso = ip_off + offsetof(struct ip, ip_sum); - /* - * Start offset for payload checksum calculation. - * End offset for payload checksum calculation. - * Offset of place to put the checksum. - */ - TXD->upper_setup.tcp_fields.tucss = ip_off + (ip->ip_hl << 2); - TXD->upper_setup.tcp_fields.tucse = 0; - TXD->upper_setup.tcp_fields.tucso = - ip_off + (ip->ip_hl << 2) + offsetof(struct tcphdr, th_sum); - /* - * Payload size per packet w/o any headers. - * Length of all headers up to payload. - */ - TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz); - TXD->tcp_seg_setup.fields.hdr_len = hdr_len; - - TXD->cmd_and_length = htole32(adapter->txd_cmd | - E1000_TXD_CMD_DEXT | /* Extended descr */ - E1000_TXD_CMD_TSE | /* TSE context */ - E1000_TXD_CMD_IP | /* Do IP csum */ - E1000_TXD_CMD_TCP | /* Do TCP checksum */ - (mp->m_pkthdr.len - (hdr_len))); /* Total len */ - - tx_buffer->m_head = NULL; - tx_buffer->next_eop = -1; - - if (++cur == adapter->num_tx_desc) - cur = 0; - - txr->tx_avail--; - txr->next_avail_desc = cur; - txr->tx_tso = TRUE; -} - - -/********************************************************************** - * - * Examine each tx_buffer in the used queue. If the hardware is done - * processing the packet then free associated resources. The - * tx_buffer is put back on the free queue. - * - **********************************************************************/ -static void -em_txeof(struct tx_ring *txr) -{ - struct adapter *adapter = txr->adapter; - int first, last, done, processed; - struct em_txbuffer *tx_buffer; - struct e1000_tx_desc *tx_desc, *eop_desc; - if_t ifp = adapter->ifp; - - EM_TX_LOCK_ASSERT(txr); -#ifdef DEV_NETMAP - if (netmap_tx_irq(ifp, txr->me)) - return; -#endif /* DEV_NETMAP */ - - /* No work, make sure hang detection is disabled */ - if (txr->tx_avail == adapter->num_tx_desc) { - txr->busy = EM_TX_IDLE; - return; - } - - processed = 0; - first = txr->next_to_clean; - tx_desc = &txr->tx_base[first]; - tx_buffer = &txr->tx_buffers[first]; - last = tx_buffer->next_eop; - eop_desc = &txr->tx_base[last]; - - /* - * What this does is get the index of the - * first descriptor AFTER the EOP of the - * first packet, that way we can do the - * simple comparison on the inner while loop. - */ - if (++last == adapter->num_tx_desc) - last = 0; - done = last; - - bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, - BUS_DMASYNC_POSTREAD); - - while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) { - /* We clean the range of the packet */ - while (first != done) { - tx_desc->upper.data = 0; - tx_desc->lower.data = 0; - tx_desc->buffer_addr = 0; - ++txr->tx_avail; - ++processed; - - if (tx_buffer->m_head) { - bus_dmamap_sync(txr->txtag, - tx_buffer->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(txr->txtag, - tx_buffer->map); - m_freem(tx_buffer->m_head); - tx_buffer->m_head = NULL; - } - tx_buffer->next_eop = -1; - - if (++first == adapter->num_tx_desc) - first = 0; - - tx_buffer = &txr->tx_buffers[first]; - tx_desc = &txr->tx_base[first]; - } - if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); - /* See if we can continue to the next packet */ - last = tx_buffer->next_eop; - if (last != -1) { - eop_desc = &txr->tx_base[last]; - /* Get new done point */ - if (++last == adapter->num_tx_desc) last = 0; - done = last; - } else - break; - } - bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - - txr->next_to_clean = first; - - /* - ** Hang detection: we know there's work outstanding - ** or the entry return would have been taken, so no - ** descriptor processed here indicates a potential hang. - ** The local timer will examine this and do a reset if needed. - */ - if (processed == 0) { - if (txr->busy != EM_TX_HUNG) - ++txr->busy; - } else /* At least one descriptor was cleaned */ - txr->busy = EM_TX_BUSY; /* note this clears HUNG */ - - /* - * If we have a minimum free, clear IFF_DRV_OACTIVE - * to tell the stack that it is OK to send packets. - * Notice that all writes of OACTIVE happen under the - * TX lock which, with a single queue, guarantees - * sanity. - */ - if (txr->tx_avail >= EM_MAX_SCATTER) { - if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); - } - - /* Disable hang detection if all clean */ - if (txr->tx_avail == adapter->num_tx_desc) - txr->busy = EM_TX_IDLE; -} - -/********************************************************************* - * - * Refresh RX descriptor mbufs from system mbuf buffer pool. - * - **********************************************************************/ -static void -em_refresh_mbufs(struct rx_ring *rxr, int limit) -{ - struct adapter *adapter = rxr->adapter; - struct mbuf *m; - bus_dma_segment_t segs; - struct em_rxbuffer *rxbuf; - int i, j, error, nsegs; - bool cleaned = FALSE; - - i = j = rxr->next_to_refresh; - /* - ** Get one descriptor beyond - ** our work mark to control - ** the loop. - */ - if (++j == adapter->num_rx_desc) - j = 0; - - while (j != limit) { - rxbuf = &rxr->rx_buffers[i]; - if (rxbuf->m_head == NULL) { - m = m_getjcl(M_NOWAIT, MT_DATA, - M_PKTHDR, adapter->rx_mbuf_sz); - /* - ** If we have a temporary resource shortage - ** that causes a failure, just abort refresh - ** for now, we will return to this point when - ** reinvoked from em_rxeof. - */ - if (m == NULL) - goto update; - } else - m = rxbuf->m_head; - - m->m_len = m->m_pkthdr.len = adapter->rx_mbuf_sz; - m->m_flags |= M_PKTHDR; - m->m_data = m->m_ext.ext_buf; - - /* Use bus_dma machinery to setup the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->rxtag, rxbuf->map, - m, &segs, &nsegs, BUS_DMA_NOWAIT); - if (error != 0) { - printf("Refresh mbufs: hdr dmamap load" - " failure - %d\n", error); - m_free(m); - rxbuf->m_head = NULL; - goto update; - } - rxbuf->m_head = m; - rxbuf->paddr = segs.ds_addr; - bus_dmamap_sync(rxr->rxtag, - rxbuf->map, BUS_DMASYNC_PREREAD); - em_setup_rxdesc(&rxr->rx_base[i], rxbuf); - cleaned = TRUE; - - i = j; /* Next is precalulated for us */ - rxr->next_to_refresh = i; - /* Calculate next controlling index */ - if (++j == adapter->num_rx_desc) - j = 0; - } -update: - /* - ** Update the tail pointer only if, - ** and as far as we have refreshed. - */ - if (cleaned) - E1000_WRITE_REG(&adapter->hw, - E1000_RDT(rxr->me), rxr->next_to_refresh); - - return; -} - - -/********************************************************************* - * - * Allocate memory for rx_buffer structures. Since we use one - * rx_buffer per received packet, the maximum number of rx_buffer's - * that we'll need is equal to the number of receive descriptors - * that we've allocated. - * - **********************************************************************/ -static int -em_allocate_receive_buffers(struct rx_ring *rxr) -{ - struct adapter *adapter = rxr->adapter; - device_t dev = adapter->dev; - struct em_rxbuffer *rxbuf; - int error; - - rxr->rx_buffers = malloc(sizeof(struct em_rxbuffer) * - adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); - if (rxr->rx_buffers == NULL) { - device_printf(dev, "Unable to allocate rx_buffer memory\n"); - return (ENOMEM); - } - - error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - MJUM9BYTES, /* maxsize */ - 1, /* nsegments */ - MJUM9BYTES, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockarg */ - &rxr->rxtag); - if (error) { - device_printf(dev, "%s: bus_dma_tag_create failed %d\n", - __func__, error); - goto fail; - } - - rxbuf = rxr->rx_buffers; - for (int i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { - rxbuf = &rxr->rx_buffers[i]; - error = bus_dmamap_create(rxr->rxtag, 0, &rxbuf->map); - if (error) { - device_printf(dev, "%s: bus_dmamap_create failed: %d\n", - __func__, error); - goto fail; - } - } - - return (0); - -fail: - em_free_receive_structures(adapter); - return (error); -} - - -/********************************************************************* - * - * Initialize a receive ring and its buffers. - * - **********************************************************************/ -static int -em_setup_receive_ring(struct rx_ring *rxr) -{ - struct adapter *adapter = rxr->adapter; - struct em_rxbuffer *rxbuf; - bus_dma_segment_t seg[1]; - int rsize, nsegs, error = 0; -#ifdef DEV_NETMAP - struct netmap_slot *slot; - struct netmap_adapter *na = netmap_getna(adapter->ifp); -#endif - - - /* Clear the ring contents */ - EM_RX_LOCK(rxr); - rsize = roundup2(adapter->num_rx_desc * - sizeof(union e1000_rx_desc_extended), EM_DBA_ALIGN); - bzero((void *)rxr->rx_base, rsize); -#ifdef DEV_NETMAP - slot = netmap_reset(na, NR_RX, rxr->me, 0); -#endif - - /* - ** Free current RX buffer structs and their mbufs - */ - for (int i = 0; i < adapter->num_rx_desc; i++) { - rxbuf = &rxr->rx_buffers[i]; - if (rxbuf->m_head != NULL) { - bus_dmamap_sync(rxr->rxtag, rxbuf->map, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->rxtag, rxbuf->map); - m_freem(rxbuf->m_head); - rxbuf->m_head = NULL; /* mark as freed */ - } - } - - /* Now replenish the mbufs */ - for (int j = 0; j != adapter->num_rx_desc; ++j) { - rxbuf = &rxr->rx_buffers[j]; -#ifdef DEV_NETMAP - if (slot) { - int si = netmap_idx_n2k(&na->rx_rings[rxr->me], j); - uint64_t paddr; - void *addr; - - addr = PNMB(na, slot + si, &paddr); - netmap_load_map(na, rxr->rxtag, rxbuf->map, addr); - rxbuf->paddr = paddr; - em_setup_rxdesc(&rxr->rx_base[j], rxbuf); - continue; - } -#endif /* DEV_NETMAP */ - rxbuf->m_head = m_getjcl(M_NOWAIT, MT_DATA, - M_PKTHDR, adapter->rx_mbuf_sz); - if (rxbuf->m_head == NULL) { - error = ENOBUFS; - goto fail; - } - rxbuf->m_head->m_len = adapter->rx_mbuf_sz; - rxbuf->m_head->m_flags &= ~M_HASFCS; /* we strip it */ - rxbuf->m_head->m_pkthdr.len = adapter->rx_mbuf_sz; - - /* Get the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->rxtag, - rxbuf->map, rxbuf->m_head, seg, - &nsegs, BUS_DMA_NOWAIT); - if (error != 0) { - m_freem(rxbuf->m_head); - rxbuf->m_head = NULL; - goto fail; - } - bus_dmamap_sync(rxr->rxtag, - rxbuf->map, BUS_DMASYNC_PREREAD); - - rxbuf->paddr = seg[0].ds_addr; - em_setup_rxdesc(&rxr->rx_base[j], rxbuf); - } - rxr->next_to_check = 0; - rxr->next_to_refresh = 0; - bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - -fail: - EM_RX_UNLOCK(rxr); - return (error); -} - -/********************************************************************* - * - * Initialize all receive rings. - * - **********************************************************************/ -static int -em_setup_receive_structures(struct adapter *adapter) -{ - struct rx_ring *rxr = adapter->rx_rings; - int q; - - for (q = 0; q < adapter->num_queues; q++, rxr++) - if (em_setup_receive_ring(rxr)) - goto fail; - - return (0); -fail: - /* - * Free RX buffers allocated so far, we will only handle - * the rings that completed, the failing case will have - * cleaned up for itself. 'q' failed, so its the terminus. - */ - for (int i = 0; i < q; ++i) { - rxr = &adapter->rx_rings[i]; - for (int n = 0; n < adapter->num_rx_desc; n++) { - struct em_rxbuffer *rxbuf; - rxbuf = &rxr->rx_buffers[n]; - if (rxbuf->m_head != NULL) { - bus_dmamap_sync(rxr->rxtag, rxbuf->map, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->rxtag, rxbuf->map); - m_freem(rxbuf->m_head); - rxbuf->m_head = NULL; - } - } - rxr->next_to_check = 0; - rxr->next_to_refresh = 0; - } - - return (ENOBUFS); -} - -/********************************************************************* - * - * Free all receive rings. - * - **********************************************************************/ -static void -em_free_receive_structures(struct adapter *adapter) -{ - struct rx_ring *rxr = adapter->rx_rings; - - for (int i = 0; i < adapter->num_queues; i++, rxr++) { - em_free_receive_buffers(rxr); - /* Free the ring memory as well */ - em_dma_free(adapter, &rxr->rxdma); - EM_RX_LOCK_DESTROY(rxr); - } - - free(adapter->rx_rings, M_DEVBUF); -} - - -/********************************************************************* - * - * Free receive ring data structures - * - **********************************************************************/ -static void -em_free_receive_buffers(struct rx_ring *rxr) -{ - struct adapter *adapter = rxr->adapter; - struct em_rxbuffer *rxbuf = NULL; - - INIT_DEBUGOUT("free_receive_buffers: begin"); - - if (rxr->rx_buffers != NULL) { - for (int i = 0; i < adapter->num_rx_desc; i++) { - rxbuf = &rxr->rx_buffers[i]; - if (rxbuf->map != NULL) { - bus_dmamap_sync(rxr->rxtag, rxbuf->map, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->rxtag, rxbuf->map); - bus_dmamap_destroy(rxr->rxtag, rxbuf->map); - } - if (rxbuf->m_head != NULL) { - m_freem(rxbuf->m_head); - rxbuf->m_head = NULL; - } - } - free(rxr->rx_buffers, M_DEVBUF); - rxr->rx_buffers = NULL; - rxr->next_to_check = 0; - rxr->next_to_refresh = 0; - } - - if (rxr->rxtag != NULL) { - bus_dma_tag_destroy(rxr->rxtag); - rxr->rxtag = NULL; - } - - return; -} - - /********************************************************************* * * Enable receive unit. @@ -4552,12 +2916,15 @@ em_free_receive_buffers(struct rx_ring *rxr) **********************************************************************/ static void -em_initialize_receive_unit(struct adapter *adapter) +em_initialize_receive_unit(if_ctx_t ctx) { - struct rx_ring *rxr = adapter->rx_rings; - if_t ifp = adapter->ifp; + struct adapter *adapter = iflib_get_softc(ctx); + if_softc_ctx_t scctx = adapter->shared; + struct ifnet *ifp = iflib_get_ifp(ctx); struct e1000_hw *hw = &adapter->hw; - u32 rctl, rxcsum, rfctl; + struct em_rx_queue *que; + int i; + u32 rctl, rxcsum, rfctl; INIT_DEBUGOUT("em_initialize_receive_units: begin"); @@ -4585,28 +2952,30 @@ em_initialize_receive_unit(struct adapter *adapter) else rctl &= ~E1000_RCTL_LPE; - /* Strip the CRC */ - if (!em_disable_crc_stripping) + /* Strip the CRC */ + if (!em_disable_crc_stripping) rctl |= E1000_RCTL_SECRC; - E1000_WRITE_REG(&adapter->hw, E1000_RADV, - adapter->rx_abs_int_delay.value); + if (adapter->hw.mac.type >= e1000_82540) { + E1000_WRITE_REG(&adapter->hw, E1000_RADV, + adapter->rx_abs_int_delay.value); + /* + * Set the interrupt throttling rate. Value is calculated + * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) + */ + E1000_WRITE_REG(hw, E1000_ITR, DEFAULT_ITR); + } E1000_WRITE_REG(&adapter->hw, E1000_RDTR, adapter->rx_int_delay.value); - /* - * Set the interrupt throttling rate. Value is calculated - * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) - */ - E1000_WRITE_REG(hw, E1000_ITR, DEFAULT_ITR); /* Use extended rx descriptor formats */ rfctl = E1000_READ_REG(hw, E1000_RFCTL); rfctl |= E1000_RFCTL_EXTEN; /* - ** When using MSIX interrupts we need to throttle - ** using the EITR register (82574 only) - */ + * When using MSIX interrupts we need to throttle + * using the EITR register (82574 only) + */ if (hw->mac.type == e1000_82574) { for (int i = 0; i < 4; i++) E1000_WRITE_REG(hw, E1000_EITR_82574(i), @@ -4617,92 +2986,62 @@ em_initialize_receive_unit(struct adapter *adapter) E1000_WRITE_REG(hw, E1000_RFCTL, rfctl); rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); - if (if_getcapenable(ifp) & IFCAP_RXCSUM) { -#ifdef EM_MULTIQUEUE - rxcsum |= E1000_RXCSUM_TUOFL | - E1000_RXCSUM_IPOFL | - E1000_RXCSUM_PCSD; -#else - rxcsum |= E1000_RXCSUM_TUOFL; -#endif + if (if_getcapenable(ifp) & IFCAP_RXCSUM && + adapter->hw.mac.type >= e1000_82543) { + if (adapter->tx_num_queues > 1) { + if (adapter->hw.mac.type >= igb_mac_min) { + rxcsum |= E1000_RXCSUM_PCSD; + if (hw->mac.type != e1000_82575) + rxcsum |= E1000_RXCSUM_CRCOFL; + } else + rxcsum |= E1000_RXCSUM_TUOFL | + E1000_RXCSUM_IPOFL | + E1000_RXCSUM_PCSD; + } else { + if (adapter->hw.mac.type >= igb_mac_min) + rxcsum |= E1000_RXCSUM_IPPCSE; + else + rxcsum |= E1000_RXCSUM_TUOFL | E1000_RXCSUM_IPOFL; + if (adapter->hw.mac.type > e1000_82575) + rxcsum |= E1000_RXCSUM_CRCOFL; + } } else rxcsum &= ~E1000_RXCSUM_TUOFL; E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); -#ifdef EM_MULTIQUEUE -#define RSSKEYLEN 10 - if (adapter->num_queues > 1) { - uint8_t rss_key[4 * RSSKEYLEN]; - uint32_t reta = 0; - int i; - - /* - * Configure RSS key - */ - arc4rand(rss_key, sizeof(rss_key), 0); - for (i = 0; i < RSSKEYLEN; ++i) { - uint32_t rssrk = 0; - - rssrk = EM_RSSRK_VAL(rss_key, i); - E1000_WRITE_REG(hw,E1000_RSSRK(i), rssrk); - } - - /* - * Configure RSS redirect table in following fashion: - * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)] - */ - for (i = 0; i < sizeof(reta); ++i) { - uint32_t q; - - q = (i % adapter->num_queues) << 7; - reta |= q << (8 * i); - } - - for (i = 0; i < 32; ++i) { - E1000_WRITE_REG(hw, E1000_RETA(i), reta); - } - - E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_RSS_ENABLE_2Q | - E1000_MRQC_RSS_FIELD_IPV4_TCP | - E1000_MRQC_RSS_FIELD_IPV4 | - E1000_MRQC_RSS_FIELD_IPV6_TCP_EX | - E1000_MRQC_RSS_FIELD_IPV6_EX | - E1000_MRQC_RSS_FIELD_IPV6); + if (adapter->rx_num_queues > 1) { + if (adapter->hw.mac.type >= igb_mac_min) + igb_initialize_rss_mapping(adapter); + else + em_initialize_rss_mapping(adapter); } -#endif + /* - ** XXX TEMPORARY WORKAROUND: on some systems with 82573 - ** long latencies are observed, like Lenovo X60. This - ** change eliminates the problem, but since having positive - ** values in RDTR is a known source of problems on other - ** platforms another solution is being sought. - */ + * XXX TEMPORARY WORKAROUND: on some systems with 82573 + * long latencies are observed, like Lenovo X60. This + * change eliminates the problem, but since having positive + * values in RDTR is a known source of problems on other + * platforms another solution is being sought. + */ if (hw->mac.type == e1000_82573) E1000_WRITE_REG(hw, E1000_RDTR, 0x20); - for (int i = 0; i < adapter->num_queues; i++, rxr++) { + for (i = 0, que = adapter->rx_queues; i < adapter->rx_num_queues; i++, que++) { + struct rx_ring *rxr = &que->rxr; /* Setup the Base and Length of the Rx Descriptor Ring */ - u64 bus_addr = rxr->rxdma.dma_paddr; - u32 rdt = adapter->num_rx_desc - 1; /* default */ + u64 bus_addr = rxr->rx_paddr; +#if 0 + u32 rdt = adapter->rx_num_queues -1; /* default */ +#endif E1000_WRITE_REG(hw, E1000_RDLEN(i), - adapter->num_rx_desc * sizeof(union e1000_rx_desc_extended)); + scctx->isc_nrxd[0] * sizeof(union e1000_rx_desc_extended)); E1000_WRITE_REG(hw, E1000_RDBAH(i), (u32)(bus_addr >> 32)); E1000_WRITE_REG(hw, E1000_RDBAL(i), (u32)bus_addr); /* Setup the Head and Tail Descriptor Pointers */ E1000_WRITE_REG(hw, E1000_RDH(i), 0); -#ifdef DEV_NETMAP - /* - * an init() while a netmap client is active must - * preserve the rx buffers passed to userspace. - */ - if (if_getcapenable(ifp) & IFCAP_NETMAP) { - struct netmap_adapter *na = netmap_getna(adapter->ifp); - rdt -= nm_kr_rxspace(&na->rx_rings[i]); - } -#endif /* DEV_NETMAP */ - E1000_WRITE_REG(hw, E1000_RDT(i), rdt); + E1000_WRITE_REG(hw, E1000_RDT(i), 0); } /* @@ -4712,6 +3051,7 @@ em_initialize_receive_unit(struct adapter *adapter) * Only write to RXDCTL(1) if there is a need for different * settings. */ + if (((adapter->hw.mac.type == e1000_ich9lan) || (adapter->hw.mac.type == e1000_pch2lan) || (adapter->hw.mac.type == e1000_ich10lan)) && @@ -4719,384 +3059,151 @@ em_initialize_receive_unit(struct adapter *adapter) u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0)); E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl | 3); } else if (adapter->hw.mac.type == e1000_82574) { - for (int i = 0; i < adapter->num_queues; i++) { + for (int i = 0; i < adapter->rx_num_queues; i++) { u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); - rxdctl |= 0x20; /* PTHRESH */ rxdctl |= 4 << 8; /* HTHRESH */ rxdctl |= 4 << 16;/* WTHRESH */ rxdctl |= 1 << 24; /* Switch to granularity */ E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); } - } - - if (adapter->hw.mac.type >= e1000_pch2lan) { - if (if_getmtu(ifp) > ETHERMTU) - e1000_lv_jumbo_workaround_ich8lan(hw, TRUE); - else - e1000_lv_jumbo_workaround_ich8lan(hw, FALSE); - } - - /* Make sure VLAN Filters are off */ - rctl &= ~E1000_RCTL_VFE; - - if (adapter->rx_mbuf_sz == MCLBYTES) - rctl |= E1000_RCTL_SZ_2048; - else if (adapter->rx_mbuf_sz == MJUMPAGESIZE) - rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX; - else if (adapter->rx_mbuf_sz > MJUMPAGESIZE) - rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX; - - /* ensure we clear use DTYPE of 00 here */ - rctl &= ~0x00000C00; - /* Write out the settings */ - E1000_WRITE_REG(hw, E1000_RCTL, rctl); - - return; -} - - -/********************************************************************* - * - * This routine executes in interrupt context. It replenishes - * the mbufs in the descriptor and sends data which has been - * dma'ed into host memory to upper layer. - * - * We loop at most count times if count is > 0, or until done if - * count < 0. - * - * For polling we also now return the number of cleaned packets - *********************************************************************/ -static bool -em_rxeof(struct rx_ring *rxr, int count, int *done) -{ - struct adapter *adapter = rxr->adapter; - if_t ifp = adapter->ifp; - struct mbuf *mp, *sendmp; - u32 status = 0; - u16 len; - int i, processed, rxdone = 0; - bool eop; - union e1000_rx_desc_extended *cur; - - EM_RX_LOCK(rxr); - - /* Sync the ring */ - bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - - -#ifdef DEV_NETMAP - if (netmap_rx_irq(ifp, rxr->me, &processed)) { - EM_RX_UNLOCK(rxr); - return (FALSE); - } -#endif /* DEV_NETMAP */ - - for (i = rxr->next_to_check, processed = 0; count != 0;) { - if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) - break; - - cur = &rxr->rx_base[i]; - status = le32toh(cur->wb.upper.status_error); - mp = sendmp = NULL; - - if ((status & E1000_RXD_STAT_DD) == 0) - break; - - len = le16toh(cur->wb.upper.length); - eop = (status & E1000_RXD_STAT_EOP) != 0; - - if ((status & E1000_RXDEXT_ERR_FRAME_ERR_MASK) || - (rxr->discard == TRUE)) { - adapter->dropped_pkts++; - ++rxr->rx_discarded; - if (!eop) /* Catch subsequent segs */ - rxr->discard = TRUE; - else - rxr->discard = FALSE; - em_rx_discard(rxr, i); - goto next_desc; - } - bus_dmamap_unload(rxr->rxtag, rxr->rx_buffers[i].map); - - /* Assign correct length to the current fragment */ - mp = rxr->rx_buffers[i].m_head; - mp->m_len = len; + } else if (adapter->hw.mac.type >= igb_mac_min) { + u32 psize, srrctl = 0; - /* Trigger for refresh */ - rxr->rx_buffers[i].m_head = NULL; - - /* First segment? */ - if (rxr->fmp == NULL) { - mp->m_pkthdr.len = len; - rxr->fmp = rxr->lmp = mp; - } else { - /* Chain mbuf's together */ - mp->m_flags &= ~M_PKTHDR; - rxr->lmp->m_next = mp; - rxr->lmp = mp; - rxr->fmp->m_pkthdr.len += len; - } - - if (eop) { - --count; - sendmp = rxr->fmp; - if_setrcvif(sendmp, ifp); - if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); - em_receive_checksum(status, sendmp); -#ifndef __NO_STRICT_ALIGNMENT - if (adapter->hw.mac.max_frame_size > - (MCLBYTES - ETHER_ALIGN) && - em_fixup_rx(rxr) != 0) - goto skip; -#endif - if (status & E1000_RXD_STAT_VP) { - if_setvtag(sendmp, - le16toh(cur->wb.upper.vlan)); - sendmp->m_flags |= M_VLANTAG; + if (if_getmtu(ifp) > ETHERMTU) { + /* Set maximum packet len */ + if (adapter->rx_mbuf_sz <= 4096) { + srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT; + rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX; + } else if (adapter->rx_mbuf_sz > 4096) { + srrctl |= 8192 >> E1000_SRRCTL_BSIZEPKT_SHIFT; + rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX; } -#ifndef __NO_STRICT_ALIGNMENT -skip: -#endif - rxr->fmp = rxr->lmp = NULL; + psize = scctx->isc_max_frame_size; + /* are we on a vlan? */ + if (ifp->if_vlantrunk != NULL) + psize += VLAN_TAG_SIZE; + E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize); + } else { + srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT; + rctl |= E1000_RCTL_SZ_2048; } -next_desc: - /* Sync the ring */ - bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - - /* Zero out the receive descriptors status. */ - cur->wb.upper.status_error &= htole32(~0xFF); - ++rxdone; /* cumulative for POLL */ - ++processed; - - /* Advance our pointers to the next descriptor. */ - if (++i == adapter->num_rx_desc) - i = 0; - - /* Send to the stack */ - if (sendmp != NULL) { - rxr->next_to_check = i; - EM_RX_UNLOCK(rxr); - if_input(ifp, sendmp); - EM_RX_LOCK(rxr); - i = rxr->next_to_check; + + /* + * If TX flow control is disabled and there's >1 queue defined, + * enable DROP. + * + * This drops frames rather than hanging the RX MAC for all queues. + */ + if ((adapter->rx_num_queues > 1) && + (adapter->fc == e1000_fc_none || + adapter->fc == e1000_fc_rx_pause)) { + srrctl |= E1000_SRRCTL_DROP_EN; } + /* Setup the Base and Length of the Rx Descriptor Rings */ + for (i = 0, que = adapter->rx_queues; i < adapter->rx_num_queues; i++, que++) { + struct rx_ring *rxr = &que->rxr; + u64 bus_addr = rxr->rx_paddr; + u32 rxdctl; + +#ifdef notyet + /* Configure for header split? -- ignore for now */ + rxr->hdr_split = igb_header_split; +#else + srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF; +#endif - /* Only refresh mbufs every 8 descriptors */ - if (processed == 8) { - em_refresh_mbufs(rxr, i); - processed = 0; - } + E1000_WRITE_REG(hw, E1000_RDLEN(i), + scctx->isc_nrxd[0] * sizeof(struct e1000_rx_desc)); + E1000_WRITE_REG(hw, E1000_RDBAH(i), + (uint32_t)(bus_addr >> 32)); + E1000_WRITE_REG(hw, E1000_RDBAL(i), + (uint32_t)bus_addr); + E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl); + /* Enable this Queue */ + rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); + rxdctl |= E1000_RXDCTL_QUEUE_ENABLE; + rxdctl &= 0xFFF00000; + rxdctl |= IGB_RX_PTHRESH; + rxdctl |= IGB_RX_HTHRESH << 8; + rxdctl |= IGB_RX_WTHRESH << 16; + E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); + } + } else if (adapter->hw.mac.type >= e1000_pch2lan) { + if (if_getmtu(ifp) > ETHERMTU) + e1000_lv_jumbo_workaround_ich8lan(hw, TRUE); + else + e1000_lv_jumbo_workaround_ich8lan(hw, FALSE); } - /* Catch any remaining refresh work */ - if (e1000_rx_unrefreshed(rxr)) - em_refresh_mbufs(rxr, i); - - rxr->next_to_check = i; - if (done != NULL) - *done = rxdone; - EM_RX_UNLOCK(rxr); - - return ((status & E1000_RXD_STAT_DD) ? TRUE : FALSE); -} - -static __inline void -em_rx_discard(struct rx_ring *rxr, int i) -{ - struct em_rxbuffer *rbuf; + /* Make sure VLAN Filters are off */ + rctl &= ~E1000_RCTL_VFE; - rbuf = &rxr->rx_buffers[i]; - bus_dmamap_unload(rxr->rxtag, rbuf->map); + if (adapter->hw.mac.type < igb_mac_min) { + if (adapter->rx_mbuf_sz == MCLBYTES) + rctl |= E1000_RCTL_SZ_2048; + else if (adapter->rx_mbuf_sz == MJUMPAGESIZE) + rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX; + else if (adapter->rx_mbuf_sz > MJUMPAGESIZE) + rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX; - /* Free any previous pieces */ - if (rxr->fmp != NULL) { - rxr->fmp->m_flags |= M_PKTHDR; - m_freem(rxr->fmp); - rxr->fmp = NULL; - rxr->lmp = NULL; - } - /* - ** Free buffer and allow em_refresh_mbufs() - ** to clean up and recharge buffer. - */ - if (rbuf->m_head) { - m_free(rbuf->m_head); - rbuf->m_head = NULL; - } - return; -} - -#ifndef __NO_STRICT_ALIGNMENT -/* - * When jumbo frames are enabled we should realign entire payload on - * architecures with strict alignment. This is serious design mistake of 8254x - * as it nullifies DMA operations. 8254x just allows RX buffer size to be - * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its - * payload. On architecures without strict alignment restrictions 8254x still - * performs unaligned memory access which would reduce the performance too. - * To avoid copying over an entire frame to align, we allocate a new mbuf and - * copy ethernet header to the new mbuf. The new mbuf is prepended into the - * existing mbuf chain. - * - * Be aware, best performance of the 8254x is achived only when jumbo frame is - * not used at all on architectures with strict alignment. - */ -static int -em_fixup_rx(struct rx_ring *rxr) -{ - struct adapter *adapter = rxr->adapter; - struct mbuf *m, *n; - int error; - - error = 0; - m = rxr->fmp; - if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) { - bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len); - m->m_data += ETHER_HDR_LEN; - } else { - MGETHDR(n, M_NOWAIT, MT_DATA); - if (n != NULL) { - bcopy(m->m_data, n->m_data, ETHER_HDR_LEN); - m->m_data += ETHER_HDR_LEN; - m->m_len -= ETHER_HDR_LEN; - n->m_len = ETHER_HDR_LEN; - M_MOVE_PKTHDR(n, m); - n->m_next = m; - rxr->fmp = n; - } else { - adapter->dropped_pkts++; - m_freem(rxr->fmp); - rxr->fmp = NULL; - error = ENOMEM; - } + /* ensure we clear use DTYPE of 00 here */ + rctl &= ~0x00000C00; } - return (error); -} -#endif - -static void -em_setup_rxdesc(union e1000_rx_desc_extended *rxd, const struct em_rxbuffer *rxbuf) -{ - rxd->read.buffer_addr = htole64(rxbuf->paddr); - /* DD bits must be cleared */ - rxd->wb.upper.status_error= 0; -} - -/********************************************************************* - * - * Verify that the hardware indicated that the checksum is valid. - * Inform the stack about the status of checksum so that stack - * doesn't spend time verifying the checksum. - * - *********************************************************************/ -static void -em_receive_checksum(uint32_t status, struct mbuf *mp) -{ - mp->m_pkthdr.csum_flags = 0; - - /* Ignore Checksum bit is set */ - if (status & E1000_RXD_STAT_IXSM) - return; - - /* If the IP checksum exists and there is no IP Checksum error */ - if ((status & (E1000_RXD_STAT_IPCS | E1000_RXDEXT_STATERR_IPE)) == - E1000_RXD_STAT_IPCS) { - mp->m_pkthdr.csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID); - } + /* Write out the settings */ + E1000_WRITE_REG(hw, E1000_RCTL, rctl); - /* TCP or UDP checksum */ - if ((status & (E1000_RXD_STAT_TCPCS | E1000_RXDEXT_STATERR_TCPE)) == - E1000_RXD_STAT_TCPCS) { - mp->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); - mp->m_pkthdr.csum_data = htons(0xffff); - } - if (status & E1000_RXD_STAT_UDPCS) { - mp->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); - mp->m_pkthdr.csum_data = htons(0xffff); - } + return; } -/* - * This routine is run via an vlan - * config EVENT - */ static void -em_register_vlan(void *arg, if_t ifp, u16 vtag) +em_if_vlan_register(if_ctx_t ctx, u16 vtag) { - struct adapter *adapter = if_getsoftc(ifp); - u32 index, bit; - - if ((void*)adapter != arg) /* Not our event */ - return; + struct adapter *adapter = iflib_get_softc(ctx); + u32 index, bit; - if ((vtag == 0) || (vtag > 4095)) /* Invalid ID */ - return; - - EM_CORE_LOCK(adapter); index = (vtag >> 5) & 0x7F; bit = vtag & 0x1F; adapter->shadow_vfta[index] |= (1 << bit); ++adapter->num_vlans; - /* Re-init to load the changes */ - if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) - em_init_locked(adapter); - EM_CORE_UNLOCK(adapter); } -/* - * This routine is run via an vlan - * unconfig EVENT - */ static void -em_unregister_vlan(void *arg, if_t ifp, u16 vtag) +em_if_vlan_unregister(if_ctx_t ctx, u16 vtag) { - struct adapter *adapter = if_getsoftc(ifp); - u32 index, bit; - - if (adapter != arg) - return; - - if ((vtag == 0) || (vtag > 4095)) /* Invalid */ - return; + struct adapter *adapter = iflib_get_softc(ctx); + u32 index, bit; - EM_CORE_LOCK(adapter); index = (vtag >> 5) & 0x7F; bit = vtag & 0x1F; adapter->shadow_vfta[index] &= ~(1 << bit); --adapter->num_vlans; - /* Re-init to load the changes */ - if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) - em_init_locked(adapter); - EM_CORE_UNLOCK(adapter); } static void em_setup_vlan_hw_support(struct adapter *adapter) { struct e1000_hw *hw = &adapter->hw; - u32 reg; + u32 reg; /* - ** We get here thru init_locked, meaning - ** a soft reset, this has already cleared - ** the VFTA and other state, so if there - ** have been no vlan's registered do nothing. - */ + * We get here thru init_locked, meaning + * a soft reset, this has already cleared + * the VFTA and other state, so if there + * have been no vlan's registered do nothing. + */ if (adapter->num_vlans == 0) - return; + return; /* - ** A soft reset zero's out the VFTA, so - ** we need to repopulate it now. - */ + * A soft reset zero's out the VFTA, so + * we need to repopulate it now. + */ for (int i = 0; i < EM_VFTA_SIZE; i++) - if (adapter->shadow_vfta[i] != 0) + if (adapter->shadow_vfta[i] != 0) E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, - i, adapter->shadow_vfta[i]); + i, adapter->shadow_vfta[i]); reg = E1000_READ_REG(hw, E1000_CTRL); reg |= E1000_CTRL_VME; @@ -5110,25 +3217,38 @@ em_setup_vlan_hw_support(struct adapter *adapter) } static void -em_enable_intr(struct adapter *adapter) +em_if_enable_intr(if_ctx_t ctx) { + struct adapter *adapter = iflib_get_softc(ctx); struct e1000_hw *hw = &adapter->hw; u32 ims_mask = IMS_ENABLE_MASK; if (hw->mac.type == e1000_82574) { - E1000_WRITE_REG(hw, EM_EIAC, adapter->ims); + E1000_WRITE_REG(hw, EM_EIAC, EM_MSIX_MASK); ims_mask |= adapter->ims; - } + } else if (adapter->intr_type == IFLIB_INTR_MSIX && hw->mac.type >= igb_mac_min) { + u32 mask = (adapter->que_mask | adapter->link_mask); + + E1000_WRITE_REG(&adapter->hw, E1000_EIAC, mask); + E1000_WRITE_REG(&adapter->hw, E1000_EIAM, mask); + E1000_WRITE_REG(&adapter->hw, E1000_EIMS, mask); + ims_mask = E1000_IMS_LSC; + } + E1000_WRITE_REG(hw, E1000_IMS, ims_mask); } static void -em_disable_intr(struct adapter *adapter) +em_if_disable_intr(if_ctx_t ctx) { + struct adapter *adapter = iflib_get_softc(ctx); struct e1000_hw *hw = &adapter->hw; - if (hw->mac.type == e1000_82574) - E1000_WRITE_REG(hw, EM_EIAC, 0); + if (adapter->intr_type == IFLIB_INTR_MSIX) { + if (hw->mac.type >= igb_mac_min) + E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0); + E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0); + } E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); } @@ -5149,7 +3269,7 @@ em_init_manageability(struct adapter *adapter) /* disable hardware interception of ARP */ manc &= ~(E1000_MANC_ARP_EN); - /* enable receiving management packets to the host */ + /* enable receiving management packets to the host */ manc |= E1000_MANC_EN_MNG2HOST; #define E1000_MNG2HOST_PORT_623 (1 << 5) #define E1000_MNG2HOST_PORT_664 (1 << 6) @@ -5247,19 +3367,38 @@ em_is_valid_ether_addr(u8 *addr) ** later use. */ static void -em_get_wakeup(device_t dev) +em_get_wakeup(if_ctx_t ctx) { - struct adapter *adapter = device_get_softc(dev); - u16 eeprom_data = 0, device_id, apme_mask; + struct adapter *adapter = iflib_get_softc(ctx); + device_t dev = iflib_get_dev(ctx); + u16 eeprom_data = 0, device_id, apme_mask; adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw); apme_mask = EM_EEPROM_APME; switch (adapter->hw.mac.type) { + case e1000_82542: + case e1000_82543: + break; + case e1000_82544: + e1000_read_nvm(&adapter->hw, + NVM_INIT_CONTROL2_REG, 1, &eeprom_data); + apme_mask = EM_82544_APME; + break; + case e1000_82546: + case e1000_82546_rev_3: + if (adapter->hw.bus.func == 1) { + e1000_read_nvm(&adapter->hw, + NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); + break; + } else + e1000_read_nvm(&adapter->hw, + NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); + break; case e1000_82573: case e1000_82583: adapter->has_amt = TRUE; - /* Falls thru */ + /* FALLTHROUGH */ case e1000_82571: case e1000_82572: case e1000_80003es2lan: @@ -5278,6 +3417,15 @@ em_get_wakeup(device_t dev) case e1000_pch2lan: case e1000_pch_lpt: case e1000_pch_spt: + case e1000_82575: /* listing all igb devices */ + case e1000_82576: + case e1000_82580: + case e1000_i350: + case e1000_i354: + case e1000_i210: + case e1000_i211: + case e1000_vfadapt: + case e1000_vfadapt_i350: apme_mask = E1000_WUC_APME; adapter->has_amt = TRUE; eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC); @@ -5290,12 +3438,31 @@ em_get_wakeup(device_t dev) if (eeprom_data & apme_mask) adapter->wol = (E1000_WUFC_MAG | E1000_WUFC_MC); /* - * We have the eeprom settings, now apply the special cases - * where the eeprom may be wrong or the board won't support - * wake on lan on a particular port + * We have the eeprom settings, now apply the special cases + * where the eeprom may be wrong or the board won't support + * wake on lan on a particular port */ device_id = pci_get_device(dev); - switch (device_id) { + switch (device_id) { + case E1000_DEV_ID_82546GB_PCIE: + adapter->wol = 0; + break; + case E1000_DEV_ID_82546EB_FIBER: + case E1000_DEV_ID_82546GB_FIBER: + /* Wake events only supported on port A for dual fiber + * regardless of eeprom setting */ + if (E1000_READ_REG(&adapter->hw, E1000_STATUS) & + E1000_STATUS_FUNC_1) + adapter->wol = 0; + break; + case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3: + /* if quad port adapter, disable WoL on all but port A */ + if (global_quad_port_a != 0) + adapter->wol = 0; + /* Reset for multiple quad port adapters */ + if (++global_quad_port_a == 4) + global_quad_port_a = 0; + break; case E1000_DEV_ID_82571EB_FIBER: /* Wake events only supported on port A for dual fiber * regardless of eeprom setting */ @@ -5306,13 +3473,13 @@ em_get_wakeup(device_t dev) case E1000_DEV_ID_82571EB_QUAD_COPPER: case E1000_DEV_ID_82571EB_QUAD_FIBER: case E1000_DEV_ID_82571EB_QUAD_COPPER_LP: - /* if quad port adapter, disable WoL on all but port A */ + /* if quad port adapter, disable WoL on all but port A */ if (global_quad_port_a != 0) adapter->wol = 0; /* Reset for multiple quad port adapters */ if (++global_quad_port_a == 4) global_quad_port_a = 0; - break; + break; } return; } @@ -5322,12 +3489,13 @@ em_get_wakeup(device_t dev) * Enable PCI Wake On Lan capability */ static void -em_enable_wakeup(device_t dev) +em_enable_wakeup(if_ctx_t ctx) { - struct adapter *adapter = device_get_softc(dev); - if_t ifp = adapter->ifp; - u32 pmc, ctrl, ctrl_ext, rctl, wuc; - u16 status; + struct adapter *adapter = iflib_get_softc(ctx); + device_t dev = iflib_get_dev(ctx); + if_t ifp = iflib_get_ifp(ctx); + u32 pmc, ctrl, ctrl_ext, rctl, wuc; + u16 status; if ((pci_find_cap(dev, PCIY_PMG, &pmc) != 0)) return; @@ -5337,8 +3505,8 @@ em_enable_wakeup(device_t dev) ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3); E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); wuc = E1000_READ_REG(&adapter->hw, E1000_WUC); - wuc |= E1000_WUC_PME_EN; - E1000_WRITE_REG(&adapter->hw, E1000_WUC, wuc); + wuc |= (E1000_WUC_PME_EN | E1000_WUC_APME); + E1000_WRITE_REG(&adapter->hw, E1000_WUC, wuc); if ((adapter->hw.mac.type == e1000_ich8lan) || (adapter->hw.mac.type == e1000_pchlan) || @@ -5355,12 +3523,15 @@ em_enable_wakeup(device_t dev) } /* - ** Determine type of Wakeup: note that wol - ** is set with all bits on by default. - */ + * Determine type of Wakeup: note that wol + * is set with all bits on by default. + */ if ((if_getcapenable(ifp) & IFCAP_WOL_MAGIC) == 0) adapter->wol &= ~E1000_WUFC_MAG; + if ((if_getcapenable(ifp) & IFCAP_WOL_UCAST) == 0) + adapter->wol &= ~E1000_WUFC_EX; + if ((if_getcapenable(ifp) & IFCAP_WOL_MCAST) == 0) adapter->wol &= ~E1000_WUFC_MC; else { @@ -5369,10 +3540,7 @@ em_enable_wakeup(device_t dev) E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); } - if ((adapter->hw.mac.type == e1000_pchlan) || - (adapter->hw.mac.type == e1000_pch2lan) || - (adapter->hw.mac.type == e1000_pch_lpt) || - (adapter->hw.mac.type == e1000_pch_spt)) { + if ( adapter->hw.mac.type >= e1000_pchlan) { if (em_enable_phy_wakeup(adapter)) return; } else { @@ -5383,20 +3551,20 @@ em_enable_wakeup(device_t dev) if (adapter->hw.phy.type == e1000_phy_igp_3) e1000_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw); - /* Request PME */ - status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2); + /* Request PME */ + status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2); status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE); if (if_getcapenable(ifp) & IFCAP_WOL) status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; - pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2); + pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2); return; } /* -** WOL in the newer chipset interfaces (pchlan) -** require thing to be copied into the phy -*/ + * WOL in the newer chipset interfaces (pchlan) + * require thing to be copied into the phy + */ static int em_enable_phy_wakeup(struct adapter *adapter) { @@ -5437,7 +3605,7 @@ em_enable_phy_wakeup(struct adapter *adapter) /* enable PHY wakeup in MAC register */ E1000_WRITE_REG(hw, E1000_WUC, - E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN); + E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN | E1000_WUC_APME); E1000_WRITE_REG(hw, E1000_WUFC, adapter->wol); /* configure and enable PHY wakeup in PHY registers */ @@ -5468,11 +3636,10 @@ out: } static void -em_led_func(void *arg, int onoff) +em_if_led_func(if_ctx_t ctx, int onoff) { - struct adapter *adapter = arg; + struct adapter *adapter = iflib_get_softc(ctx); - EM_CORE_LOCK(adapter); if (onoff) { e1000_setup_led(&adapter->hw); e1000_led_on(&adapter->hw); @@ -5480,26 +3647,25 @@ em_led_func(void *arg, int onoff) e1000_led_off(&adapter->hw); e1000_cleanup_led(&adapter->hw); } - EM_CORE_UNLOCK(adapter); } /* -** Disable the L0S and L1 LINK states -*/ + * Disable the L0S and L1 LINK states + */ static void em_disable_aspm(struct adapter *adapter) { - int base, reg; - u16 link_cap,link_ctrl; - device_t dev = adapter->dev; + int base, reg; + u16 link_cap,link_ctrl; + device_t dev = adapter->dev; switch (adapter->hw.mac.type) { - case e1000_82573: - case e1000_82574: - case e1000_82583: - break; - default: - return; + case e1000_82573: + case e1000_82574: + case e1000_82583: + break; + default: + return; } if (pci_find_cap(dev, PCIY_EXPRESS, &base) != 0) return; @@ -5595,27 +3761,26 @@ em_update_stats_counters(struct adapter *adapter) adapter->stats.icrxoc += E1000_READ_REG(&adapter->hw, E1000_ICRXOC); if (adapter->hw.mac.type >= e1000_82543) { - adapter->stats.algnerrc += + adapter->stats.algnerrc += E1000_READ_REG(&adapter->hw, E1000_ALGNERRC); - adapter->stats.rxerrc += + adapter->stats.rxerrc += E1000_READ_REG(&adapter->hw, E1000_RXERRC); - adapter->stats.tncrs += + adapter->stats.tncrs += E1000_READ_REG(&adapter->hw, E1000_TNCRS); - adapter->stats.cexterr += + adapter->stats.cexterr += E1000_READ_REG(&adapter->hw, E1000_CEXTERR); - adapter->stats.tsctc += + adapter->stats.tsctc += E1000_READ_REG(&adapter->hw, E1000_TSCTC); - adapter->stats.tsctfc += + adapter->stats.tsctfc += E1000_READ_REG(&adapter->hw, E1000_TSCTFC); } } static uint64_t -em_get_counter(if_t ifp, ift_counter cnt) +em_if_get_counter(if_ctx_t ctx, ift_counter cnt) { - struct adapter *adapter; - - adapter = if_getsoftc(ifp); + struct adapter *adapter = iflib_get_softc(ctx); + struct ifnet *ifp = iflib_get_ifp(ctx); switch (cnt) { case IFCOUNTER_COLLISIONS: @@ -5651,10 +3816,9 @@ em_sysctl_reg_handler(SYSCTL_HANDLER_ARGS) static void em_add_hw_stats(struct adapter *adapter) { - device_t dev = adapter->dev; - - struct tx_ring *txr = adapter->tx_rings; - struct rx_ring *rxr = adapter->rx_rings; + device_t dev = iflib_get_dev(adapter->ctx); + struct em_tx_queue *tx_que = adapter->tx_queues; + struct em_rx_queue *rx_que = adapter->rx_queues; struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); struct sysctl_oid *tree = device_get_sysctl_tree(dev); @@ -5666,18 +3830,18 @@ em_add_hw_stats(struct adapter *adapter) #define QUEUE_NAME_LEN 32 char namebuf[QUEUE_NAME_LEN]; - + /* Driver Statistics */ - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", + SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", CTLFLAG_RD, &adapter->dropped_pkts, "Driver dropped packets"); SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq", CTLFLAG_RD, &adapter->link_irq, "Link MSIX IRQ Handled"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_fail", + SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_fail", CTLFLAG_RD, &adapter->mbuf_defrag_failed, "Defragmenting mbuf chain failed"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail", + SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail", CTLFLAG_RD, &adapter->no_tx_dma_setup, "Driver tx dma failure in xmit"); SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns", @@ -5686,7 +3850,6 @@ em_add_hw_stats(struct adapter *adapter) SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts", CTLFLAG_RD, &adapter->watchdog_events, "Watchdog timeouts"); - SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control", CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_CTRL, em_sysctl_reg_handler, "IU", @@ -5698,44 +3861,48 @@ em_add_hw_stats(struct adapter *adapter) SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water", CTLFLAG_RD, &adapter->hw.fc.high_water, 0, "Flow Control High Watermark"); - SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", + SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", CTLFLAG_RD, &adapter->hw.fc.low_water, 0, "Flow Control Low Watermark"); - for (int i = 0; i < adapter->num_queues; i++, txr++, rxr++) { + for (int i = 0; i < adapter->tx_num_queues; i++, tx_que++) { + struct tx_ring *txr = &tx_que->txr; snprintf(namebuf, QUEUE_NAME_LEN, "queue_tx_%d", i); queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, NULL, "TX Queue Name"); queue_list = SYSCTL_CHILDREN(queue_node); - SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head", + SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head", CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDH(txr->me), em_sysctl_reg_handler, "IU", - "Transmit Descriptor Head"); - SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail", + "Transmit Descriptor Head"); + SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail", CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDT(txr->me), em_sysctl_reg_handler, "IU", - "Transmit Descriptor Tail"); + "Transmit Descriptor Tail"); SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tx_irq", CTLFLAG_RD, &txr->tx_irq, "Queue MSI-X Transmit Interrupts"); - SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_desc_avail", + SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_desc_avail", CTLFLAG_RD, &txr->no_desc_avail, "Queue No Descriptor Available"); + } - snprintf(namebuf, QUEUE_NAME_LEN, "queue_rx_%d", i); + for (int j = 0; j < adapter->rx_num_queues; j++, rx_que++) { + struct rx_ring *rxr = &rx_que->rxr; + snprintf(namebuf, QUEUE_NAME_LEN, "queue_rx_%d", j); queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, NULL, "RX Queue Name"); queue_list = SYSCTL_CHILDREN(queue_node); - SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head", + SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head", CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDH(rxr->me), em_sysctl_reg_handler, "IU", "Receive Descriptor Head"); - SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail", + SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail", CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDT(rxr->me), em_sysctl_reg_handler, "IU", @@ -5747,7 +3914,7 @@ em_add_hw_stats(struct adapter *adapter) /* MAC stats get their own sub node */ - stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", + stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", CTLFLAG_RD, NULL, "Statistics"); stat_list = SYSCTL_CHILDREN(stat_node); @@ -5850,14 +4017,14 @@ em_add_hw_stats(struct adapter *adapter) SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522", CTLFLAG_RD, &adapter->stats.prc1522, "1023-1522 byte frames received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", - CTLFLAG_RD, &adapter->stats.gorc, - "Good Octets Received"); + SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", + CTLFLAG_RD, &adapter->stats.gorc, + "Good Octets Received"); /* Packet Transmission Stats */ - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", - CTLFLAG_RD, &adapter->stats.gotc, - "Good Octets Transmitted"); + SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", + CTLFLAG_RD, &adapter->stats.gotc, + "Good Octets Transmitted"); SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd", CTLFLAG_RD, &adapter->stats.tpt, "Total Packets Transmitted"); @@ -5973,8 +4140,8 @@ em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS) static void em_print_nvm_info(struct adapter *adapter) { - u16 eeprom_data; - int i, j, row = 0; + u16 eeprom_data; + int i, j, row = 0; /* Its a bit crude, but it gets the job done */ printf("\nInterface EEPROM Dump:\n"); @@ -5998,7 +4165,7 @@ em_sysctl_int_delay(SYSCTL_HANDLER_ARGS) u32 regval; int error, usecs, ticks; - info = (struct em_int_delay_info *)arg1; + info = (struct em_int_delay_info *) arg1; usecs = info->value; error = sysctl_handle_int(oidp, &usecs, 0, req); if (error != 0 || req->newptr == NULL) @@ -6011,8 +4178,7 @@ em_sysctl_int_delay(SYSCTL_HANDLER_ARGS) ticks *= 4; adapter = info->adapter; - - EM_CORE_LOCK(adapter); + regval = E1000_READ_OFFSET(&adapter->hw, info->offset); regval = (regval & ~0xffff) | (ticks & 0xffff); /* Handle a few special cases. */ @@ -6029,7 +4195,6 @@ em_sysctl_int_delay(SYSCTL_HANDLER_ARGS) break; } E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval); - EM_CORE_UNLOCK(adapter); return (0); } @@ -6047,78 +4212,66 @@ em_add_int_delay_sysctl(struct adapter *adapter, const char *name, info, 0, em_sysctl_int_delay, "I", description); } -static void -em_set_sysctl_value(struct adapter *adapter, const char *name, - const char *description, int *limit, int value) -{ - *limit = value; - SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), - OID_AUTO, name, CTLFLAG_RW, limit, value, description); -} - - /* -** Set flow control using sysctl: -** Flow control values: -** 0 - off -** 1 - rx pause -** 2 - tx pause -** 3 - full -*/ + * Set flow control using sysctl: + * Flow control values: + * 0 - off + * 1 - rx pause + * 2 - tx pause + * 3 - full + */ static int em_set_flowcntl(SYSCTL_HANDLER_ARGS) -{ - int error; - static int input = 3; /* default is full */ - struct adapter *adapter = (struct adapter *) arg1; - - error = sysctl_handle_int(oidp, &input, 0, req); - - if ((error) || (req->newptr == NULL)) - return (error); - +{ + int error; + static int input = 3; /* default is full */ + struct adapter *adapter = (struct adapter *) arg1; + + error = sysctl_handle_int(oidp, &input, 0, req); + + if ((error) || (req->newptr == NULL)) + return (error); + if (input == adapter->fc) /* no change? */ return (error); - switch (input) { - case e1000_fc_rx_pause: - case e1000_fc_tx_pause: - case e1000_fc_full: - case e1000_fc_none: - adapter->hw.fc.requested_mode = input; - adapter->fc = input; - break; - default: - /* Do nothing */ - return (error); - } + switch (input) { + case e1000_fc_rx_pause: + case e1000_fc_tx_pause: + case e1000_fc_full: + case e1000_fc_none: + adapter->hw.fc.requested_mode = input; + adapter->fc = input; + break; + default: + /* Do nothing */ + return (error); + } - adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode; - e1000_force_mac_fc(&adapter->hw); - return (error); + adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode; + e1000_force_mac_fc(&adapter->hw); + return (error); } /* -** Manage Energy Efficient Ethernet: -** Control values: -** 0/1 - enabled/disabled -*/ + * Manage Energy Efficient Ethernet: + * Control values: + * 0/1 - enabled/disabled + */ static int em_sysctl_eee(SYSCTL_HANDLER_ARGS) { - struct adapter *adapter = (struct adapter *) arg1; - int error, value; - - value = adapter->hw.dev_spec.ich8lan.eee_disable; - error = sysctl_handle_int(oidp, &value, 0, req); - if (error || req->newptr == NULL) - return (error); - EM_CORE_LOCK(adapter); - adapter->hw.dev_spec.ich8lan.eee_disable = (value != 0); - em_init_locked(adapter); - EM_CORE_UNLOCK(adapter); - return (0); + struct adapter *adapter = (struct adapter *) arg1; + int error, value; + + value = adapter->hw.dev_spec.ich8lan.eee_disable; + error = sysctl_handle_int(oidp, &value, 0, req); + if (error || req->newptr == NULL) + return (error); + adapter->hw.dev_spec.ich8lan.eee_disable = (value != 0); + em_if_init(adapter->ctx); + + return (0); } static int @@ -6135,66 +4288,84 @@ em_sysctl_debug_info(SYSCTL_HANDLER_ARGS) return (error); if (result == 1) { - adapter = (struct adapter *)arg1; + adapter = (struct adapter *) arg1; em_print_debug_info(adapter); } return (error); } +static int +em_get_rs(SYSCTL_HANDLER_ARGS) +{ + struct adapter *adapter = (struct adapter *) arg1; + int error; + int result; + + result = 0; + error = sysctl_handle_int(oidp, &result, 0, req); + + if (error || !req->newptr || result != 1) + return (error); + em_dump_rs(adapter); + + return (error); +} + +static void +em_if_debug(if_ctx_t ctx) +{ + em_dump_rs(iflib_get_softc(ctx)); +} + /* -** This routine is meant to be fluid, add whatever is -** needed for debugging a problem. -jfv -*/ + * This routine is meant to be fluid, add whatever is + * needed for debugging a problem. -jfv + */ static void em_print_debug_info(struct adapter *adapter) { - device_t dev = adapter->dev; - struct tx_ring *txr = adapter->tx_rings; - struct rx_ring *rxr = adapter->rx_rings; + device_t dev = iflib_get_dev(adapter->ctx); + struct ifnet *ifp = iflib_get_ifp(adapter->ctx); + struct tx_ring *txr = &adapter->tx_queues->txr; + struct rx_ring *rxr = &adapter->rx_queues->rxr; - if (if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) printf("Interface is RUNNING "); else printf("Interface is NOT RUNNING\n"); - if (if_getdrvflags(adapter->ifp) & IFF_DRV_OACTIVE) + if (if_getdrvflags(ifp) & IFF_DRV_OACTIVE) printf("and INACTIVE\n"); else printf("and ACTIVE\n"); - for (int i = 0; i < adapter->num_queues; i++, txr++, rxr++) { + for (int i = 0; i < adapter->tx_num_queues; i++, txr++) { device_printf(dev, "TX Queue %d ------\n", i); device_printf(dev, "hw tdh = %d, hw tdt = %d\n", - E1000_READ_REG(&adapter->hw, E1000_TDH(i)), - E1000_READ_REG(&adapter->hw, E1000_TDT(i))); - device_printf(dev, "Tx Queue Status = %d\n", txr->busy); - device_printf(dev, "TX descriptors avail = %d\n", - txr->tx_avail); - device_printf(dev, "Tx Descriptors avail failure = %ld\n", - txr->no_desc_avail); - device_printf(dev, "RX Queue %d ------\n", i); + E1000_READ_REG(&adapter->hw, E1000_TDH(i)), + E1000_READ_REG(&adapter->hw, E1000_TDT(i))); + + } + for (int j=0; j < adapter->rx_num_queues; j++, rxr++) { + device_printf(dev, "RX Queue %d ------\n", j); device_printf(dev, "hw rdh = %d, hw rdt = %d\n", - E1000_READ_REG(&adapter->hw, E1000_RDH(i)), - E1000_READ_REG(&adapter->hw, E1000_RDT(i))); - device_printf(dev, "RX discarded packets = %ld\n", - rxr->rx_discarded); - device_printf(dev, "RX Next to Check = %d\n", rxr->next_to_check); - device_printf(dev, "RX Next to Refresh = %d\n", rxr->next_to_refresh); + E1000_READ_REG(&adapter->hw, E1000_RDH(j)), + E1000_READ_REG(&adapter->hw, E1000_RDT(j))); } } -#ifdef EM_MULTIQUEUE /* * 82574 only: * Write a new value to the EEPROM increasing the number of MSIX * vectors from 3 to 5, for proper multiqueue support. */ static void -em_enable_vectors_82574(struct adapter *adapter) +em_enable_vectors_82574(if_ctx_t ctx) { + struct adapter *adapter = iflib_get_softc(ctx); struct e1000_hw *hw = &adapter->hw; - device_t dev = adapter->dev; + device_t dev = iflib_get_dev(ctx); u16 edata; e1000_read_nvm(hw, EM_NVM_PCIE_CTRL, 1, &edata); @@ -6209,42 +4380,3 @@ em_enable_vectors_82574(struct adapter *adapter) device_printf(dev, "Writing to eeprom: done\n"); } } -#endif - -#ifdef DDB -DB_COMMAND(em_reset_dev, em_ddb_reset_dev) -{ - devclass_t dc; - int max_em; - - dc = devclass_find("em"); - max_em = devclass_get_maxunit(dc); - - for (int index = 0; index < (max_em - 1); index++) { - device_t dev; - dev = devclass_get_device(dc, index); - if (device_get_driver(dev) == &em_driver) { - struct adapter *adapter = device_get_softc(dev); - EM_CORE_LOCK(adapter); - em_init_locked(adapter); - EM_CORE_UNLOCK(adapter); - } - } -} -DB_COMMAND(em_dump_queue, em_ddb_dump_queue) -{ - devclass_t dc; - int max_em; - - dc = devclass_find("em"); - max_em = devclass_get_maxunit(dc); - - for (int index = 0; index < (max_em - 1); index++) { - device_t dev; - dev = devclass_get_device(dc, index); - if (device_get_driver(dev) == &em_driver) - em_print_debug_info(device_get_softc(dev)); - } - -} -#endif diff --git a/freebsd/sys/dev/e1000/if_em.h b/freebsd/sys/dev/e1000/if_em.h index 2a2bf2cc..eb353700 100644 --- a/freebsd/sys/dev/e1000/if_em.h +++ b/freebsd/sys/dev/e1000/if_em.h @@ -1,36 +1,92 @@ -/****************************************************************************** - - Copyright (c) 2001-2015, Intel Corporation - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - -******************************************************************************/ +/*- + * Copyright (c) 2016 Matt Macy <mmacy@nextbsd.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + /*$FreeBSD$*/ +#include <rtems/bsd/local/opt_ddb.h> +#include <rtems/bsd/local/opt_inet.h> +#include <rtems/bsd/local/opt_inet6.h> + +#ifdef HAVE_KERNEL_OPTION_HEADERS +#include <rtems/bsd/local/opt_device_polling.h> +#endif + +#include <rtems/bsd/sys/param.h> +#include <sys/systm.h> +#ifdef DDB +#include <sys/types.h> +#include <ddb/ddb.h> +#endif +#if __FreeBSD_version >= 800000 +#include <sys/buf_ring.h> +#endif +#include <sys/bus.h> +#include <sys/endian.h> +#include <sys/kernel.h> +#include <sys/kthread.h> +#include <sys/malloc.h> +#include <sys/mbuf.h> +#include <sys/module.h> +#include <sys/rman.h> +#include <sys/smp.h> +#include <sys/socket.h> +#include <sys/sockio.h> +#include <sys/sysctl.h> +#include <sys/taskqueue.h> +#include <sys/eventhandler.h> +#include <machine/bus.h> +#include <machine/resource.h> + +#include <net/bpf.h> +#include <net/ethernet.h> +#include <net/if.h> +#include <net/if_var.h> +#include <net/if_arp.h> +#include <net/if_dl.h> +#include <net/if_media.h> +#include <net/iflib.h> + +#include <net/if_types.h> +#include <net/if_vlan_var.h> + +#include <netinet/in_systm.h> +#include <netinet/in.h> +#include <netinet/if_ether.h> +#include <netinet/ip.h> +#include <netinet/ip6.h> +#include <netinet/tcp.h> +#include <netinet/udp.h> + +#include <machine/in_cksum.h> +#include <dev/led/led.h> +#include <dev/pci/pcivar.h> +#include <dev/pci/pcireg.h> + +#include "e1000_api.h" +#include "e1000_82571.h" +#include <rtems/bsd/local/ifdi_if.h> #ifndef _EM_H_DEFINED_ @@ -51,13 +107,10 @@ * desscriptors should meet the following condition. * (num_tx_desc * sizeof(struct e1000_tx_desc)) % 128 == 0 */ -#define EM_MIN_TXD 80 +#define EM_MIN_TXD 128 #define EM_MAX_TXD 4096 -#ifdef EM_MULTIQUEUE -#define EM_DEFAULT_TXD 4096 -#else -#define EM_DEFAULT_TXD 1024 -#endif +#define EM_DEFAULT_TXD 1024 +#define EM_DEFAULT_MULTI_TXD 4096 /* * EM_RXD - Maximum number of receive Descriptors @@ -72,13 +125,10 @@ * desscriptors should meet the following condition. * (num_tx_desc * sizeof(struct e1000_tx_desc)) % 128 == 0 */ -#define EM_MIN_RXD 80 +#define EM_MIN_RXD 128 #define EM_MAX_RXD 4096 -#ifdef EM_MULTIQUEUE -#define EM_DEFAULT_RXD 4096 -#else -#define EM_DEFAULT_RXD 1024 -#endif +#define EM_DEFAULT_RXD 1024 +#define EM_DEFAULT_MULTI_RXD 4096 /* * EM_TIDV - Transmit Interrupt Delay Value @@ -125,11 +175,7 @@ * restoring the network connection. To eliminate the potential * for the hang ensure that EM_RDTR is set to 0. */ -#ifdef EM_MULTIQUEUE -#define EM_RDTR 64 -#else #define EM_RDTR 0 -#endif /* * Receive Interrupt Absolute Delay Timer (Not valid for 82542/82543/82544) @@ -142,22 +188,7 @@ * along with EM_RDTR, may improve traffic throughput in specific network * conditions. */ -#ifdef EM_MULTIQUEUE -#define EM_RADV 128 -#else #define EM_RADV 64 -#endif - -/* - * This parameter controls the max duration of transmit watchdog. - */ -#define EM_WATCHDOG (10 * hz) - -/* - * This parameter controls when the driver calls the routine to reclaim - * transmit descriptors. - */ -#define EM_TX_CLEANUP_THRESHOLD (adapter->num_tx_desc / 8) /* * This parameter controls whether or not autonegotation is enabled. @@ -221,6 +252,18 @@ #define PCICFG_DESC_RING_STATUS 0xe4 #define FLUSH_DESC_REQUIRED 0x100 + +#define IGB_RX_PTHRESH ((hw->mac.type == e1000_i354) ? 12 : \ + ((hw->mac.type <= e1000_82576) ? 16 : 8)) +#define IGB_RX_HTHRESH 8 +#define IGB_RX_WTHRESH ((hw->mac.type == e1000_82576 && \ + (adapter->intr_type == IFLIB_INTR_MSIX)) ? 1 : 4) + +#define IGB_TX_PTHRESH ((hw->mac.type == e1000_i354) ? 20 : 8) +#define IGB_TX_HTHRESH 1 +#define IGB_TX_WTHRESH ((hw->mac.type != e1000_82575 && \ + (adapter->intr_type == IFLIB_INTR_MSIX) ? 1 : 16) + /* * TDBA/RDBA should be aligned on 16 byte boundary. But TDLEN/RDLEN should be * multiple of 128 bytes. So we align TDBA/RDBA on 128 byte boundary. This will @@ -242,6 +285,7 @@ #define EM_BAR_TYPE(v) ((v) & EM_BAR_TYPE_MASK) #define EM_BAR_TYPE_MASK 0x00000001 #define EM_BAR_TYPE_MMEM 0x00000000 +#define EM_BAR_TYPE_IO 0x00000001 #define EM_BAR_TYPE_FLASH 0x0014 #define EM_BAR_MEM_TYPE(v) ((v) & EM_BAR_MEM_TYPE_MASK) #define EM_BAR_MEM_TYPE_MASK 0x00000006 @@ -277,7 +321,11 @@ #define EM_MSIX_LINK 0x01000000 /* For 82574 use */ #define ETH_ZLEN 60 #define ETH_ADDR_LEN 6 -#define CSUM_OFFLOAD 7 /* Offload bits in mbuf flag */ +#define EM_CSUM_OFFLOAD 7 /* Offload bits in mbuf flag */ +#define IGB_CSUM_OFFLOAD 0x0E0F /* Offload bits in mbuf flag */ + +#define IGB_PKTTYPE_MASK 0x0000FFF0 +#define IGB_DMCTLX_DCFLUSH_DIS 0x80000000 /* Disable DMA Coalesce Flush */ /* * 82574 has a nonstandard address for EIAC @@ -295,19 +343,6 @@ #define EM_NVM_MSIX_N_MASK (0x7 << EM_NVM_MSIX_N_SHIFT) #define EM_NVM_MSIX_N_SHIFT 7 -/* - * Bus dma allocation structure used by - * e1000_dma_malloc and e1000_dma_free. - */ -struct em_dma_alloc { - bus_addr_t dma_paddr; - caddr_t dma_vaddr; - bus_dma_tag_t dma_tag; - bus_dmamap_t dma_map; - bus_dma_segment_t dma_seg; - int dma_nseg; -}; - struct adapter; struct em_int_delay_info { @@ -321,35 +356,31 @@ struct em_int_delay_info { */ struct tx_ring { struct adapter *adapter; - struct mtx tx_mtx; - char mtx_name[16]; - u32 me; - u32 msix; - u32 ims; - int busy; - struct em_dma_alloc txdma; struct e1000_tx_desc *tx_base; - struct task tx_task; - struct taskqueue *tq; - u32 next_avail_desc; - u32 next_to_clean; - struct em_txbuffer *tx_buffers; - volatile u16 tx_avail; - u32 tx_tso; /* last tx was tso */ - u16 last_hw_offload; - u8 last_hw_ipcso; - u8 last_hw_ipcss; - u8 last_hw_tucso; - u8 last_hw_tucss; -#if __FreeBSD_version >= 800000 - struct buf_ring *br; -#endif + uint64_t tx_paddr; + qidx_t *tx_rsq; + bool tx_tso; /* last tx was tso */ + uint8_t me; + qidx_t tx_rs_cidx; + qidx_t tx_rs_pidx; + qidx_t tx_cidx_processed; /* Interrupt resources */ - bus_dma_tag_t txtag; void *tag; struct resource *res; unsigned long tx_irq; unsigned long no_desc_avail; + + /* Saved csum offloading context information */ + int csum_flags; + int csum_lhlen; + int csum_iphlen; + + int csum_thlen; + int csum_mss; + int csum_pktlen; + + uint32_t csum_txd_upper; + uint32_t csum_txd_lower; /* last field */ }; /* @@ -357,26 +388,15 @@ struct tx_ring { */ struct rx_ring { struct adapter *adapter; + struct em_rx_queue *que; u32 me; - u32 msix; - u32 ims; - struct mtx rx_mtx; - char mtx_name[16]; u32 payload; - struct task rx_task; - struct taskqueue *tq; union e1000_rx_desc_extended *rx_base; - struct em_dma_alloc rxdma; - u32 next_to_refresh; - u32 next_to_check; - struct em_rxbuffer *rx_buffers; - struct mbuf *fmp; - struct mbuf *lmp; + uint64_t rx_paddr; /* Interrupt resources */ void *tag; struct resource *res; - bus_dma_tag_t rxtag; bool discard; /* Soft stats */ @@ -386,62 +406,68 @@ struct rx_ring { unsigned long rx_bytes; }; +struct em_tx_queue { + struct adapter *adapter; + u32 msix; + u32 eims; /* This queue's EIMS bit */ + u32 me; + struct tx_ring txr; +}; + +struct em_rx_queue { + struct adapter *adapter; + u32 me; + u32 msix; + u32 eims; + struct rx_ring rxr; + u64 irqs; + struct if_irq que_irq; +}; /* Our adapter structure */ struct adapter { - if_t ifp; + struct ifnet *ifp; struct e1000_hw hw; + if_softc_ctx_t shared; + if_ctx_t ctx; +#define tx_num_queues shared->isc_ntxqsets +#define rx_num_queues shared->isc_nrxqsets +#define intr_type shared->isc_intr /* FreeBSD operating-system-specific structures. */ struct e1000_osdep osdep; device_t dev; struct cdev *led_dev; + struct em_tx_queue *tx_queues; + struct em_rx_queue *rx_queues; + struct if_irq irq; + struct resource *memory; struct resource *flash; - struct resource *msix_mem; + struct resource *ioport; + int io_rid; struct resource *res; void *tag; u32 linkvec; u32 ivars; - struct ifmedia media; - struct callout timer; + struct ifmedia *media; int msix; int if_flags; - int max_frame_size; int min_frame_size; - struct mtx core_mtx; int em_insert_vlan_header; u32 ims; bool in_detach; /* Task for FAST handling */ - struct task link_task; - struct task que_task; - struct taskqueue *tq; /* private task queue */ - - eventhandler_tag vlan_attach; - eventhandler_tag vlan_detach; - - u16 num_vlans; - u8 num_queues; - - /* - * Transmit rings: - * Allocated at run time, an array of rings. - */ - struct tx_ring *tx_rings; - int num_tx_desc; + struct grouptask link_task; + + u16 num_vlans; u32 txd_cmd; - /* - * Receive rings: - * Allocated at run time, an array of rings. - */ - struct rx_ring *rx_rings; - int num_rx_desc; + u32 tx_process_limit; u32 rx_process_limit; u32 rx_mbuf_sz; @@ -467,7 +493,12 @@ struct adapter { u16 link_speed; u16 link_duplex; u32 smartspeed; + u32 dmac; + int link_mask; + + u64 que_mask; + struct em_int_delay_info tx_int_delay; struct em_int_delay_info tx_abs_int_delay; struct em_int_delay_info rx_int_delay; @@ -501,34 +532,7 @@ typedef struct _em_vendor_info_t { unsigned int index; } em_vendor_info_t; -struct em_txbuffer { - int next_eop; /* Index of the desc to watch */ - struct mbuf *m_head; - bus_dmamap_t map; /* bus_dma map for packet */ -}; - -struct em_rxbuffer { - int next_eop; /* Index of the desc to watch */ - struct mbuf *m_head; - bus_dmamap_t map; /* bus_dma map for packet */ - bus_addr_t paddr; -}; - - -/* -** Find the number of unrefreshed RX descriptors -*/ -static inline u16 -e1000_rx_unrefreshed(struct rx_ring *rxr) -{ - struct adapter *adapter = rxr->adapter; - - if (rxr->next_to_check > rxr->next_to_refresh) - return (rxr->next_to_check - rxr->next_to_refresh - 1); - else - return ((adapter->num_rx_desc + rxr->next_to_check) - - rxr->next_to_refresh - 1); -} +void em_dump_rs(struct adapter *); #define EM_CORE_LOCK_INIT(_sc, _name) \ mtx_init(&(_sc)->core_mtx, _name, "EM Core Lock", MTX_DEF) diff --git a/freebsd/sys/dev/e1000/if_igb.c b/freebsd/sys/dev/e1000/if_igb.c deleted file mode 100644 index 2e9c7259..00000000 --- a/freebsd/sys/dev/e1000/if_igb.c +++ /dev/null @@ -1,6452 +0,0 @@ -#include <machine/rtems-bsd-kernel-space.h> - -/****************************************************************************** - - Copyright (c) 2001-2015, Intel Corporation - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - -******************************************************************************/ -/*$FreeBSD$*/ - - -#include <rtems/bsd/local/opt_inet.h> -#include <rtems/bsd/local/opt_inet6.h> -#include <rtems/bsd/local/opt_rss.h> - -#ifdef HAVE_KERNEL_OPTION_HEADERS -#include <rtems/bsd/local/opt_device_polling.h> -#include <rtems/bsd/local/opt_altq.h> -#endif - -#include "if_igb.h" - -/********************************************************************* - * Driver version: - *********************************************************************/ -char igb_driver_version[] = "2.5.3-k"; - - -/********************************************************************* - * PCI Device ID Table - * - * Used by probe to select devices to load on - * Last field stores an index into e1000_strings - * Last entry must be all 0s - * - * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } - *********************************************************************/ - -static igb_vendor_info_t igb_vendor_info_array[] = -{ - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_COPPER, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_FIBER_SERDES, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575GB_QUAD_COPPER, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS_SERDES, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_FIBER, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES_QUAD, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER_ET2, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_FIBER, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SERDES, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SGMII, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER_DUAL, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_QUAD_FIBER, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SERDES, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SGMII, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SFP, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_BACKPLANE, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_COPPER, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_FIBER, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SERDES, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SGMII, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_IT, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_OEM1, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_FLASHLESS, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES_FLASHLESS, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_FIBER, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SGMII, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I211_COPPER, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_1GBPS, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_SGMII, 0, 0, 0}, - /* required last entry */ - {0, 0, 0, 0, 0} -}; - -/********************************************************************* - * Table of branding strings for all supported NICs. - *********************************************************************/ - -static char *igb_strings[] = { - "Intel(R) PRO/1000 Network Connection" -}; - -/********************************************************************* - * Function prototypes - *********************************************************************/ -static int igb_probe(device_t); -static int igb_attach(device_t); -static int igb_detach(device_t); -static int igb_shutdown(device_t); -static int igb_suspend(device_t); -static int igb_resume(device_t); -#ifndef IGB_LEGACY_TX -static int igb_mq_start(struct ifnet *, struct mbuf *); -static int igb_mq_start_locked(struct ifnet *, struct tx_ring *); -static void igb_qflush(struct ifnet *); -static void igb_deferred_mq_start(void *, int); -#else -static void igb_start(struct ifnet *); -static void igb_start_locked(struct tx_ring *, struct ifnet *ifp); -#endif -static int igb_ioctl(struct ifnet *, u_long, caddr_t); -static uint64_t igb_get_counter(if_t, ift_counter); -static void igb_init(void *); -static void igb_init_locked(struct adapter *); -static void igb_stop(void *); -static void igb_media_status(struct ifnet *, struct ifmediareq *); -static int igb_media_change(struct ifnet *); -static void igb_identify_hardware(struct adapter *); -static int igb_allocate_pci_resources(struct adapter *); -static int igb_allocate_msix(struct adapter *); -static int igb_allocate_legacy(struct adapter *); -static int igb_setup_msix(struct adapter *); -static void igb_free_pci_resources(struct adapter *); -static void igb_local_timer(void *); -static void igb_reset(struct adapter *); -static int igb_setup_interface(device_t, struct adapter *); -static int igb_allocate_queues(struct adapter *); -static void igb_configure_queues(struct adapter *); - -static int igb_allocate_transmit_buffers(struct tx_ring *); -static void igb_setup_transmit_structures(struct adapter *); -static void igb_setup_transmit_ring(struct tx_ring *); -static void igb_initialize_transmit_units(struct adapter *); -static void igb_free_transmit_structures(struct adapter *); -static void igb_free_transmit_buffers(struct tx_ring *); - -static int igb_allocate_receive_buffers(struct rx_ring *); -static int igb_setup_receive_structures(struct adapter *); -static int igb_setup_receive_ring(struct rx_ring *); -static void igb_initialize_receive_units(struct adapter *); -static void igb_free_receive_structures(struct adapter *); -static void igb_free_receive_buffers(struct rx_ring *); -static void igb_free_receive_ring(struct rx_ring *); - -static void igb_enable_intr(struct adapter *); -static void igb_disable_intr(struct adapter *); -static void igb_update_stats_counters(struct adapter *); -static bool igb_txeof(struct tx_ring *); - -static __inline void igb_rx_discard(struct rx_ring *, int); -static __inline void igb_rx_input(struct rx_ring *, - struct ifnet *, struct mbuf *, u32); - -static bool igb_rxeof(struct igb_queue *, int, int *); -static void igb_rx_checksum(u32, struct mbuf *, u32); -static int igb_tx_ctx_setup(struct tx_ring *, - struct mbuf *, u32 *, u32 *); -static int igb_tso_setup(struct tx_ring *, - struct mbuf *, u32 *, u32 *); -static void igb_set_promisc(struct adapter *); -static void igb_disable_promisc(struct adapter *); -static void igb_set_multi(struct adapter *); -static void igb_update_link_status(struct adapter *); -static void igb_refresh_mbufs(struct rx_ring *, int); - -static void igb_register_vlan(void *, struct ifnet *, u16); -static void igb_unregister_vlan(void *, struct ifnet *, u16); -static void igb_setup_vlan_hw_support(struct adapter *); - -static int igb_xmit(struct tx_ring *, struct mbuf **); -static int igb_dma_malloc(struct adapter *, bus_size_t, - struct igb_dma_alloc *, int); -static void igb_dma_free(struct adapter *, struct igb_dma_alloc *); -static int igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS); -static void igb_print_nvm_info(struct adapter *); -static int igb_is_valid_ether_addr(u8 *); -static void igb_add_hw_stats(struct adapter *); - -static void igb_vf_init_stats(struct adapter *); -static void igb_update_vf_stats_counters(struct adapter *); - -/* Management and WOL Support */ -static void igb_init_manageability(struct adapter *); -static void igb_release_manageability(struct adapter *); -static void igb_get_hw_control(struct adapter *); -static void igb_release_hw_control(struct adapter *); -static void igb_enable_wakeup(device_t); -static void igb_led_func(void *, int); - -static int igb_irq_fast(void *); -static void igb_msix_que(void *); -static void igb_msix_link(void *); -static void igb_handle_que(void *context, int pending); -static void igb_handle_link(void *context, int pending); -static void igb_handle_link_locked(struct adapter *); - -static void igb_set_sysctl_value(struct adapter *, const char *, - const char *, int *, int); -static int igb_set_flowcntl(SYSCTL_HANDLER_ARGS); -static int igb_sysctl_dmac(SYSCTL_HANDLER_ARGS); -static int igb_sysctl_eee(SYSCTL_HANDLER_ARGS); - -#ifdef DEVICE_POLLING -static poll_handler_t igb_poll; -#endif /* POLLING */ - -/********************************************************************* - * FreeBSD Device Interface Entry Points - *********************************************************************/ - -static device_method_t igb_methods[] = { - /* Device interface */ - DEVMETHOD(device_probe, igb_probe), - DEVMETHOD(device_attach, igb_attach), - DEVMETHOD(device_detach, igb_detach), - DEVMETHOD(device_shutdown, igb_shutdown), - DEVMETHOD(device_suspend, igb_suspend), - DEVMETHOD(device_resume, igb_resume), - DEVMETHOD_END -}; - -static driver_t igb_driver = { - "igb", igb_methods, sizeof(struct adapter), -}; - -static devclass_t igb_devclass; -DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0); -MODULE_DEPEND(igb, pci, 1, 1, 1); -MODULE_DEPEND(igb, ether, 1, 1, 1); -#ifdef DEV_NETMAP -MODULE_DEPEND(igb, netmap, 1, 1, 1); -#endif /* DEV_NETMAP */ - -/********************************************************************* - * Tunable default values. - *********************************************************************/ - -static SYSCTL_NODE(_hw, OID_AUTO, igb, CTLFLAG_RD, 0, "IGB driver parameters"); - -/* Descriptor defaults */ -static int igb_rxd = IGB_DEFAULT_RXD; -static int igb_txd = IGB_DEFAULT_TXD; -SYSCTL_INT(_hw_igb, OID_AUTO, rxd, CTLFLAG_RDTUN, &igb_rxd, 0, - "Number of receive descriptors per queue"); -SYSCTL_INT(_hw_igb, OID_AUTO, txd, CTLFLAG_RDTUN, &igb_txd, 0, - "Number of transmit descriptors per queue"); - -/* -** AIM: Adaptive Interrupt Moderation -** which means that the interrupt rate -** is varied over time based on the -** traffic for that interrupt vector -*/ -static int igb_enable_aim = TRUE; -SYSCTL_INT(_hw_igb, OID_AUTO, enable_aim, CTLFLAG_RWTUN, &igb_enable_aim, 0, - "Enable adaptive interrupt moderation"); - -/* - * MSIX should be the default for best performance, - * but this allows it to be forced off for testing. - */ -static int igb_enable_msix = 1; -SYSCTL_INT(_hw_igb, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &igb_enable_msix, 0, - "Enable MSI-X interrupts"); - -/* -** Tuneable Interrupt rate -*/ -static int igb_max_interrupt_rate = 8000; -SYSCTL_INT(_hw_igb, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN, - &igb_max_interrupt_rate, 0, "Maximum interrupts per second"); - -#ifndef IGB_LEGACY_TX -/* -** Tuneable number of buffers in the buf-ring (drbr_xxx) -*/ -static int igb_buf_ring_size = IGB_BR_SIZE; -SYSCTL_INT(_hw_igb, OID_AUTO, buf_ring_size, CTLFLAG_RDTUN, - &igb_buf_ring_size, 0, "Size of the bufring"); -#endif - -/* -** Header split causes the packet header to -** be dma'd to a separate mbuf from the payload. -** this can have memory alignment benefits. But -** another plus is that small packets often fit -** into the header and thus use no cluster. Its -** a very workload dependent type feature. -*/ -static int igb_header_split = FALSE; -SYSCTL_INT(_hw_igb, OID_AUTO, header_split, CTLFLAG_RDTUN, &igb_header_split, 0, - "Enable receive mbuf header split"); - -/* -** This will autoconfigure based on the -** number of CPUs and max supported -** MSIX messages if left at 0. -*/ -static int igb_num_queues = 0; -SYSCTL_INT(_hw_igb, OID_AUTO, num_queues, CTLFLAG_RDTUN, &igb_num_queues, 0, - "Number of queues to configure, 0 indicates autoconfigure"); - -/* -** Global variable to store last used CPU when binding queues -** to CPUs in igb_allocate_msix. Starts at CPU_FIRST and increments when a -** queue is bound to a cpu. -*/ -static int igb_last_bind_cpu = -1; - -/* How many packets rxeof tries to clean at a time */ -static int igb_rx_process_limit = 100; -SYSCTL_INT(_hw_igb, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, - &igb_rx_process_limit, 0, - "Maximum number of received packets to process at a time, -1 means unlimited"); - -/* How many packets txeof tries to clean at a time */ -static int igb_tx_process_limit = -1; -SYSCTL_INT(_hw_igb, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN, - &igb_tx_process_limit, 0, - "Maximum number of sent packets to process at a time, -1 means unlimited"); - -#ifdef DEV_NETMAP /* see ixgbe.c for details */ -#include <dev/netmap/if_igb_netmap.h> -#endif /* DEV_NETMAP */ -/********************************************************************* - * Device identification routine - * - * igb_probe determines if the driver should be loaded on - * adapter based on PCI vendor/device id of the adapter. - * - * return BUS_PROBE_DEFAULT on success, positive on failure - *********************************************************************/ - -static int -igb_probe(device_t dev) -{ - char adapter_name[256]; - uint16_t pci_vendor_id = 0; - uint16_t pci_device_id = 0; - uint16_t pci_subvendor_id = 0; - uint16_t pci_subdevice_id = 0; - igb_vendor_info_t *ent; - - INIT_DEBUGOUT("igb_probe: begin"); - - pci_vendor_id = pci_get_vendor(dev); - if (pci_vendor_id != IGB_INTEL_VENDOR_ID) - return (ENXIO); - - pci_device_id = pci_get_device(dev); - pci_subvendor_id = pci_get_subvendor(dev); - pci_subdevice_id = pci_get_subdevice(dev); - - ent = igb_vendor_info_array; - while (ent->vendor_id != 0) { - if ((pci_vendor_id == ent->vendor_id) && - (pci_device_id == ent->device_id) && - - ((pci_subvendor_id == ent->subvendor_id) || - (ent->subvendor_id == 0)) && - - ((pci_subdevice_id == ent->subdevice_id) || - (ent->subdevice_id == 0))) { - sprintf(adapter_name, "%s, Version - %s", - igb_strings[ent->index], - igb_driver_version); - device_set_desc_copy(dev, adapter_name); - return (BUS_PROBE_DEFAULT); - } - ent++; - } - return (ENXIO); -} - -/********************************************************************* - * Device initialization routine - * - * The attach entry point is called when the driver is being loaded. - * This routine identifies the type of hardware, allocates all resources - * and initializes the hardware. - * - * return 0 on success, positive on failure - *********************************************************************/ - -static int -igb_attach(device_t dev) -{ - struct adapter *adapter; - int error = 0; - u16 eeprom_data; - - INIT_DEBUGOUT("igb_attach: begin"); - - if (resource_disabled("igb", device_get_unit(dev))) { - device_printf(dev, "Disabled by device hint\n"); - return (ENXIO); - } - - adapter = device_get_softc(dev); - adapter->dev = adapter->osdep.dev = dev; - IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); - - /* SYSCTLs */ - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, - igb_sysctl_nvm_info, "I", "NVM Information"); - - igb_set_sysctl_value(adapter, "enable_aim", - "Interrupt Moderation", &adapter->enable_aim, - igb_enable_aim); - - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "fc", CTLTYPE_INT|CTLFLAG_RW, - adapter, 0, igb_set_flowcntl, "I", "Flow Control"); - - callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); - - /* Determine hardware and mac info */ - igb_identify_hardware(adapter); - - /* Setup PCI resources */ - if (igb_allocate_pci_resources(adapter)) { - device_printf(dev, "Allocation of PCI resources failed\n"); - error = ENXIO; - goto err_pci; - } - - /* Do Shared Code initialization */ - if (e1000_setup_init_funcs(&adapter->hw, TRUE)) { - device_printf(dev, "Setup of Shared code failed\n"); - error = ENXIO; - goto err_pci; - } - - e1000_get_bus_info(&adapter->hw); - - /* Sysctls for limiting the amount of work done in the taskqueues */ - igb_set_sysctl_value(adapter, "rx_processing_limit", - "max number of rx packets to process", - &adapter->rx_process_limit, igb_rx_process_limit); - - igb_set_sysctl_value(adapter, "tx_processing_limit", - "max number of tx packets to process", - &adapter->tx_process_limit, igb_tx_process_limit); - - /* - * Validate number of transmit and receive descriptors. It - * must not exceed hardware maximum, and must be multiple - * of E1000_DBA_ALIGN. - */ - if (((igb_txd * sizeof(struct e1000_tx_desc)) % IGB_DBA_ALIGN) != 0 || - (igb_txd > IGB_MAX_TXD) || (igb_txd < IGB_MIN_TXD)) { - device_printf(dev, "Using %d TX descriptors instead of %d!\n", - IGB_DEFAULT_TXD, igb_txd); - adapter->num_tx_desc = IGB_DEFAULT_TXD; - } else - adapter->num_tx_desc = igb_txd; - if (((igb_rxd * sizeof(struct e1000_rx_desc)) % IGB_DBA_ALIGN) != 0 || - (igb_rxd > IGB_MAX_RXD) || (igb_rxd < IGB_MIN_RXD)) { - device_printf(dev, "Using %d RX descriptors instead of %d!\n", - IGB_DEFAULT_RXD, igb_rxd); - adapter->num_rx_desc = IGB_DEFAULT_RXD; - } else - adapter->num_rx_desc = igb_rxd; - - adapter->hw.mac.autoneg = DO_AUTO_NEG; - adapter->hw.phy.autoneg_wait_to_complete = FALSE; - adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; - - /* Copper options */ - if (adapter->hw.phy.media_type == e1000_media_type_copper) { - adapter->hw.phy.mdix = AUTO_ALL_MODES; - adapter->hw.phy.disable_polarity_correction = FALSE; - adapter->hw.phy.ms_type = IGB_MASTER_SLAVE; - } - - /* - * Set the frame limits assuming - * standard ethernet sized frames. - */ - adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; - - /* - ** Allocate and Setup Queues - */ - if (igb_allocate_queues(adapter)) { - error = ENOMEM; - goto err_pci; - } - - /* Allocate the appropriate stats memory */ - if (adapter->vf_ifp) { - adapter->stats = - (struct e1000_vf_stats *)malloc(sizeof \ - (struct e1000_vf_stats), M_DEVBUF, M_NOWAIT | M_ZERO); - igb_vf_init_stats(adapter); - } else - adapter->stats = - (struct e1000_hw_stats *)malloc(sizeof \ - (struct e1000_hw_stats), M_DEVBUF, M_NOWAIT | M_ZERO); - if (adapter->stats == NULL) { - device_printf(dev, "Can not allocate stats memory\n"); - error = ENOMEM; - goto err_late; - } - - /* Allocate multicast array memory. */ - adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN * - MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); - if (adapter->mta == NULL) { - device_printf(dev, "Can not allocate multicast setup array\n"); - error = ENOMEM; - goto err_late; - } - - /* Some adapter-specific advanced features */ - if (adapter->hw.mac.type >= e1000_i350) { - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "dmac", CTLTYPE_INT|CTLFLAG_RW, - adapter, 0, igb_sysctl_dmac, "I", "DMA Coalesce"); - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "eee_disabled", CTLTYPE_INT|CTLFLAG_RW, - adapter, 0, igb_sysctl_eee, "I", - "Disable Energy Efficient Ethernet"); - if (adapter->hw.phy.media_type == e1000_media_type_copper) { - if (adapter->hw.mac.type == e1000_i354) - e1000_set_eee_i354(&adapter->hw, TRUE, TRUE); - else - e1000_set_eee_i350(&adapter->hw, TRUE, TRUE); - } - } - - /* - ** Start from a known state, this is - ** important in reading the nvm and - ** mac from that. - */ - e1000_reset_hw(&adapter->hw); - - /* Make sure we have a good EEPROM before we read from it */ - if (((adapter->hw.mac.type != e1000_i210) && - (adapter->hw.mac.type != e1000_i211)) && - (e1000_validate_nvm_checksum(&adapter->hw) < 0)) { - /* - ** Some PCI-E parts fail the first check due to - ** the link being in sleep state, call it again, - ** if it fails a second time its a real issue. - */ - if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { - device_printf(dev, - "The EEPROM Checksum Is Not Valid\n"); - error = EIO; - goto err_late; - } - } - - /* - ** Copy the permanent MAC address out of the EEPROM - */ - if (e1000_read_mac_addr(&adapter->hw) < 0) { - device_printf(dev, "EEPROM read error while reading MAC" - " address\n"); - error = EIO; - goto err_late; - } - - /* Check its sanity */ - if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) { - if (adapter->vf_ifp) { - u8 addr[ETHER_ADDR_LEN]; - arc4rand(&addr, sizeof(addr), 0); - addr[0] &= 0xFE; - addr[0] |= 0x02; - bcopy(addr, adapter->hw.mac.addr, sizeof(addr)); - } else { - device_printf(dev, "Invalid MAC address\n"); - error = EIO; - goto err_late; - } - } - - /* Setup OS specific network interface */ - if (igb_setup_interface(dev, adapter) != 0) - goto err_late; - - /* Now get a good starting state */ - igb_reset(adapter); - - /* Initialize statistics */ - igb_update_stats_counters(adapter); - - adapter->hw.mac.get_link_status = 1; - igb_update_link_status(adapter); - - /* Indicate SOL/IDER usage */ - if (e1000_check_reset_block(&adapter->hw)) - device_printf(dev, - "PHY reset is blocked due to SOL/IDER session.\n"); - - /* Determine if we have to control management hardware */ - adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw); - - /* - * Setup Wake-on-Lan - */ - /* APME bit in EEPROM is mapped to WUC.APME */ - eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & E1000_WUC_APME; - if (eeprom_data) - adapter->wol = E1000_WUFC_MAG; - - /* Register for VLAN events */ - adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, - igb_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); - adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, - igb_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); - - igb_add_hw_stats(adapter); - - /* Tell the stack that the interface is not active */ - adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; - adapter->ifp->if_drv_flags |= IFF_DRV_OACTIVE; - - adapter->led_dev = led_create(igb_led_func, adapter, - device_get_nameunit(dev)); - - /* - ** Configure Interrupts - */ - if ((adapter->msix > 1) && (igb_enable_msix)) - error = igb_allocate_msix(adapter); - else /* MSI or Legacy */ - error = igb_allocate_legacy(adapter); - if (error) - goto err_late; - -#ifdef DEV_NETMAP - igb_netmap_attach(adapter); -#endif /* DEV_NETMAP */ - INIT_DEBUGOUT("igb_attach: end"); - - return (0); - -err_late: - if (igb_detach(dev) == 0) /* igb_detach() already did the cleanup */ - return(error); - igb_free_transmit_structures(adapter); - igb_free_receive_structures(adapter); - igb_release_hw_control(adapter); -err_pci: - igb_free_pci_resources(adapter); - if (adapter->ifp != NULL) - if_free(adapter->ifp); - free(adapter->mta, M_DEVBUF); - IGB_CORE_LOCK_DESTROY(adapter); - - return (error); -} - -/********************************************************************* - * Device removal routine - * - * The detach entry point is called when the driver is being removed. - * This routine stops the adapter and deallocates all the resources - * that were allocated for driver operation. - * - * return 0 on success, positive on failure - *********************************************************************/ - -static int -igb_detach(device_t dev) -{ - struct adapter *adapter = device_get_softc(dev); - struct ifnet *ifp = adapter->ifp; - - INIT_DEBUGOUT("igb_detach: begin"); - - /* Make sure VLANS are not using driver */ - if (adapter->ifp->if_vlantrunk != NULL) { - device_printf(dev,"Vlan in use, detach first\n"); - return (EBUSY); - } - - ether_ifdetach(adapter->ifp); - - if (adapter->led_dev != NULL) - led_destroy(adapter->led_dev); - -#ifdef DEVICE_POLLING - if (ifp->if_capenable & IFCAP_POLLING) - ether_poll_deregister(ifp); -#endif - - IGB_CORE_LOCK(adapter); - adapter->in_detach = 1; - igb_stop(adapter); - IGB_CORE_UNLOCK(adapter); - - e1000_phy_hw_reset(&adapter->hw); - - /* Give control back to firmware */ - igb_release_manageability(adapter); - igb_release_hw_control(adapter); - - if (adapter->wol) { - E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); - E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); - igb_enable_wakeup(dev); - } - - /* Unregister VLAN events */ - if (adapter->vlan_attach != NULL) - EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); - if (adapter->vlan_detach != NULL) - EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); - - callout_drain(&adapter->timer); - -#ifdef DEV_NETMAP - netmap_detach(adapter->ifp); -#endif /* DEV_NETMAP */ - igb_free_pci_resources(adapter); - bus_generic_detach(dev); - if_free(ifp); - - igb_free_transmit_structures(adapter); - igb_free_receive_structures(adapter); - if (adapter->mta != NULL) - free(adapter->mta, M_DEVBUF); - - IGB_CORE_LOCK_DESTROY(adapter); - - return (0); -} - -/********************************************************************* - * - * Shutdown entry point - * - **********************************************************************/ - -static int -igb_shutdown(device_t dev) -{ - return igb_suspend(dev); -} - -/* - * Suspend/resume device methods. - */ -static int -igb_suspend(device_t dev) -{ - struct adapter *adapter = device_get_softc(dev); - - IGB_CORE_LOCK(adapter); - - igb_stop(adapter); - - igb_release_manageability(adapter); - igb_release_hw_control(adapter); - - if (adapter->wol) { - E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); - E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); - igb_enable_wakeup(dev); - } - - IGB_CORE_UNLOCK(adapter); - - return bus_generic_suspend(dev); -} - -static int -igb_resume(device_t dev) -{ - struct adapter *adapter = device_get_softc(dev); - struct tx_ring *txr = adapter->tx_rings; - struct ifnet *ifp = adapter->ifp; - - IGB_CORE_LOCK(adapter); - igb_init_locked(adapter); - igb_init_manageability(adapter); - - if ((ifp->if_flags & IFF_UP) && - (ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) { - for (int i = 0; i < adapter->num_queues; i++, txr++) { - IGB_TX_LOCK(txr); -#ifndef IGB_LEGACY_TX - /* Process the stack queue only if not depleted */ - if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) && - !drbr_empty(ifp, txr->br)) - igb_mq_start_locked(ifp, txr); -#else - if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) - igb_start_locked(txr, ifp); -#endif - IGB_TX_UNLOCK(txr); - } - } - IGB_CORE_UNLOCK(adapter); - - return bus_generic_resume(dev); -} - - -#ifdef IGB_LEGACY_TX - -/********************************************************************* - * Transmit entry point - * - * igb_start is called by the stack to initiate a transmit. - * The driver will remain in this routine as long as there are - * packets to transmit and transmit resources are available. - * In case resources are not available stack is notified and - * the packet is requeued. - **********************************************************************/ - -static void -igb_start_locked(struct tx_ring *txr, struct ifnet *ifp) -{ - struct adapter *adapter = ifp->if_softc; - struct mbuf *m_head; - - IGB_TX_LOCK_ASSERT(txr); - - if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != - IFF_DRV_RUNNING) - return; - if (!adapter->link_active) - return; - - /* Call cleanup if number of TX descriptors low */ - if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD) - igb_txeof(txr); - - while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { - if (txr->tx_avail <= IGB_MAX_SCATTER) { - txr->queue_status |= IGB_QUEUE_DEPLETED; - break; - } - IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); - if (m_head == NULL) - break; - /* - * Encapsulation can modify our pointer, and or make it - * NULL on failure. In that event, we can't requeue. - */ - if (igb_xmit(txr, &m_head)) { - if (m_head != NULL) - IFQ_DRV_PREPEND(&ifp->if_snd, m_head); - if (txr->tx_avail <= IGB_MAX_SCATTER) - txr->queue_status |= IGB_QUEUE_DEPLETED; - break; - } - - /* Send a copy of the frame to the BPF listener */ - ETHER_BPF_MTAP(ifp, m_head); - - /* Set watchdog on */ - txr->watchdog_time = ticks; - txr->queue_status |= IGB_QUEUE_WORKING; - } -} - -/* - * Legacy TX driver routine, called from the - * stack, always uses tx[0], and spins for it. - * Should not be used with multiqueue tx - */ -static void -igb_start(struct ifnet *ifp) -{ - struct adapter *adapter = ifp->if_softc; - struct tx_ring *txr = adapter->tx_rings; - - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - IGB_TX_LOCK(txr); - igb_start_locked(txr, ifp); - IGB_TX_UNLOCK(txr); - } - return; -} - -#else /* ~IGB_LEGACY_TX */ - -/* -** Multiqueue Transmit Entry: -** quick turnaround to the stack -** -*/ -static int -igb_mq_start(struct ifnet *ifp, struct mbuf *m) -{ - struct adapter *adapter = ifp->if_softc; - struct igb_queue *que; - struct tx_ring *txr; - int i, err = 0; -#ifdef RSS - uint32_t bucket_id; -#endif - - /* Which queue to use */ - /* - * When doing RSS, map it to the same outbound queue - * as the incoming flow would be mapped to. - * - * If everything is setup correctly, it should be the - * same bucket that the current CPU we're on is. - */ - if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { -#ifdef RSS - if (rss_hash2bucket(m->m_pkthdr.flowid, - M_HASHTYPE_GET(m), &bucket_id) == 0) { - /* XXX TODO: spit out something if bucket_id > num_queues? */ - i = bucket_id % adapter->num_queues; - } else { -#endif - i = m->m_pkthdr.flowid % adapter->num_queues; -#ifdef RSS - } -#endif - } else { - i = curcpu % adapter->num_queues; - } - txr = &adapter->tx_rings[i]; - que = &adapter->queues[i]; - - err = drbr_enqueue(ifp, txr->br, m); - if (err) - return (err); - if (IGB_TX_TRYLOCK(txr)) { - igb_mq_start_locked(ifp, txr); - IGB_TX_UNLOCK(txr); - } else - taskqueue_enqueue(que->tq, &txr->txq_task); - - return (0); -} - -static int -igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr) -{ - struct adapter *adapter = txr->adapter; - struct mbuf *next; - int err = 0, enq = 0; - - IGB_TX_LOCK_ASSERT(txr); - - if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) || - adapter->link_active == 0) - return (ENETDOWN); - - /* Process the queue */ - while ((next = drbr_peek(ifp, txr->br)) != NULL) { - if ((err = igb_xmit(txr, &next)) != 0) { - if (next == NULL) { - /* It was freed, move forward */ - drbr_advance(ifp, txr->br); - } else { - /* - * Still have one left, it may not be - * the same since the transmit function - * may have changed it. - */ - drbr_putback(ifp, txr->br, next); - } - break; - } - drbr_advance(ifp, txr->br); - enq++; - if (next->m_flags & M_MCAST && adapter->vf_ifp) - if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); - ETHER_BPF_MTAP(ifp, next); - if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) - break; - } - if (enq > 0) { - /* Set the watchdog */ - txr->queue_status |= IGB_QUEUE_WORKING; - txr->watchdog_time = ticks; - } - if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD) - igb_txeof(txr); - if (txr->tx_avail <= IGB_MAX_SCATTER) - txr->queue_status |= IGB_QUEUE_DEPLETED; - return (err); -} - -/* - * Called from a taskqueue to drain queued transmit packets. - */ -static void -igb_deferred_mq_start(void *arg, int pending) -{ - struct tx_ring *txr = arg; - struct adapter *adapter = txr->adapter; - struct ifnet *ifp = adapter->ifp; - - IGB_TX_LOCK(txr); - if (!drbr_empty(ifp, txr->br)) - igb_mq_start_locked(ifp, txr); - IGB_TX_UNLOCK(txr); -} - -/* -** Flush all ring buffers -*/ -static void -igb_qflush(struct ifnet *ifp) -{ - struct adapter *adapter = ifp->if_softc; - struct tx_ring *txr = adapter->tx_rings; - struct mbuf *m; - - for (int i = 0; i < adapter->num_queues; i++, txr++) { - IGB_TX_LOCK(txr); - while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) - m_freem(m); - IGB_TX_UNLOCK(txr); - } - if_qflush(ifp); -} -#endif /* ~IGB_LEGACY_TX */ - -/********************************************************************* - * Ioctl entry point - * - * igb_ioctl is called when the user wants to configure the - * interface. - * - * return 0 on success, positive on failure - **********************************************************************/ - -static int -igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data) -{ - struct adapter *adapter = ifp->if_softc; - struct ifreq *ifr = (struct ifreq *)data; -#if defined(INET) || defined(INET6) - struct ifaddr *ifa = (struct ifaddr *)data; -#endif - bool avoid_reset = FALSE; - int error = 0; - - if (adapter->in_detach) - return (error); - - switch (command) { - case SIOCSIFADDR: -#ifdef INET - if (ifa->ifa_addr->sa_family == AF_INET) - avoid_reset = TRUE; -#endif -#ifdef INET6 - if (ifa->ifa_addr->sa_family == AF_INET6) - avoid_reset = TRUE; -#endif - /* - ** Calling init results in link renegotiation, - ** so we avoid doing it when possible. - */ - if (avoid_reset) { - ifp->if_flags |= IFF_UP; - if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) - igb_init(adapter); -#ifdef INET - if (!(ifp->if_flags & IFF_NOARP)) - arp_ifinit(ifp, ifa); -#endif - } else - error = ether_ioctl(ifp, command, data); - break; - case SIOCSIFMTU: - { - int max_frame_size; - - IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); - - IGB_CORE_LOCK(adapter); - max_frame_size = 9234; - if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - - ETHER_CRC_LEN) { - IGB_CORE_UNLOCK(adapter); - error = EINVAL; - break; - } - - ifp->if_mtu = ifr->ifr_mtu; - adapter->max_frame_size = - ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; - if (ifp->if_drv_flags & IFF_DRV_RUNNING) - igb_init_locked(adapter); - IGB_CORE_UNLOCK(adapter); - break; - } - case SIOCSIFFLAGS: - IOCTL_DEBUGOUT("ioctl rcv'd:\ - SIOCSIFFLAGS (Set Interface Flags)"); - IGB_CORE_LOCK(adapter); - if (ifp->if_flags & IFF_UP) { - if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { - if ((ifp->if_flags ^ adapter->if_flags) & - (IFF_PROMISC | IFF_ALLMULTI)) { - igb_disable_promisc(adapter); - igb_set_promisc(adapter); - } - } else - igb_init_locked(adapter); - } else - if (ifp->if_drv_flags & IFF_DRV_RUNNING) - igb_stop(adapter); - adapter->if_flags = ifp->if_flags; - IGB_CORE_UNLOCK(adapter); - break; - case SIOCADDMULTI: - case SIOCDELMULTI: - IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - IGB_CORE_LOCK(adapter); - igb_disable_intr(adapter); - igb_set_multi(adapter); -#ifdef DEVICE_POLLING - if (!(ifp->if_capenable & IFCAP_POLLING)) -#endif - igb_enable_intr(adapter); - IGB_CORE_UNLOCK(adapter); - } - break; - case SIOCSIFMEDIA: - /* Check SOL/IDER usage */ - IGB_CORE_LOCK(adapter); - if (e1000_check_reset_block(&adapter->hw)) { - IGB_CORE_UNLOCK(adapter); - device_printf(adapter->dev, "Media change is" - " blocked due to SOL/IDER session.\n"); - break; - } - IGB_CORE_UNLOCK(adapter); - case SIOCGIFMEDIA: - IOCTL_DEBUGOUT("ioctl rcv'd: \ - SIOCxIFMEDIA (Get/Set Interface Media)"); - error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); - break; - case SIOCSIFCAP: - { - int mask, reinit; - - IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); - reinit = 0; - mask = ifr->ifr_reqcap ^ ifp->if_capenable; -#ifdef DEVICE_POLLING - if (mask & IFCAP_POLLING) { - if (ifr->ifr_reqcap & IFCAP_POLLING) { - error = ether_poll_register(igb_poll, ifp); - if (error) - return (error); - IGB_CORE_LOCK(adapter); - igb_disable_intr(adapter); - ifp->if_capenable |= IFCAP_POLLING; - IGB_CORE_UNLOCK(adapter); - } else { - error = ether_poll_deregister(ifp); - /* Enable interrupt even in error case */ - IGB_CORE_LOCK(adapter); - igb_enable_intr(adapter); - ifp->if_capenable &= ~IFCAP_POLLING; - IGB_CORE_UNLOCK(adapter); - } - } -#endif -#if __FreeBSD_version >= 1000000 - /* HW cannot turn these on/off separately */ - if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) { - ifp->if_capenable ^= IFCAP_RXCSUM; - ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; - reinit = 1; - } - if (mask & IFCAP_TXCSUM) { - ifp->if_capenable ^= IFCAP_TXCSUM; - reinit = 1; - } - if (mask & IFCAP_TXCSUM_IPV6) { - ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; - reinit = 1; - } -#else - if (mask & IFCAP_HWCSUM) { - ifp->if_capenable ^= IFCAP_HWCSUM; - reinit = 1; - } -#endif - if (mask & IFCAP_TSO4) { - ifp->if_capenable ^= IFCAP_TSO4; - reinit = 1; - } - if (mask & IFCAP_TSO6) { - ifp->if_capenable ^= IFCAP_TSO6; - reinit = 1; - } - if (mask & IFCAP_VLAN_HWTAGGING) { - ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; - reinit = 1; - } - if (mask & IFCAP_VLAN_HWFILTER) { - ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; - reinit = 1; - } - if (mask & IFCAP_VLAN_HWTSO) { - ifp->if_capenable ^= IFCAP_VLAN_HWTSO; - reinit = 1; - } - if (mask & IFCAP_LRO) { - ifp->if_capenable ^= IFCAP_LRO; - reinit = 1; - } - if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) - igb_init(adapter); - VLAN_CAPABILITIES(ifp); - break; - } - - default: - error = ether_ioctl(ifp, command, data); - break; - } - - return (error); -} - - -/********************************************************************* - * Init entry point - * - * This routine is used in two ways. It is used by the stack as - * init entry point in network interface structure. It is also used - * by the driver as a hw/sw initialization routine to get to a - * consistent state. - * - * return 0 on success, positive on failure - **********************************************************************/ - -static void -igb_init_locked(struct adapter *adapter) -{ - struct ifnet *ifp = adapter->ifp; - device_t dev = adapter->dev; - - INIT_DEBUGOUT("igb_init: begin"); - - IGB_CORE_LOCK_ASSERT(adapter); - - igb_disable_intr(adapter); - callout_stop(&adapter->timer); - - /* Get the latest mac address, User can use a LAA */ - bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr, - ETHER_ADDR_LEN); - - /* Put the address into the Receive Address Array */ - e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); - - igb_reset(adapter); - igb_update_link_status(adapter); - - E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); - - /* Set hardware offload abilities */ - ifp->if_hwassist = 0; - if (ifp->if_capenable & IFCAP_TXCSUM) { -#if __FreeBSD_version >= 1000000 - ifp->if_hwassist |= (CSUM_IP_TCP | CSUM_IP_UDP); - if (adapter->hw.mac.type != e1000_82575) - ifp->if_hwassist |= CSUM_IP_SCTP; -#else - ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); -#if __FreeBSD_version >= 800000 - if (adapter->hw.mac.type != e1000_82575) - ifp->if_hwassist |= CSUM_SCTP; -#endif -#endif - } - -#if __FreeBSD_version >= 1000000 - if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) { - ifp->if_hwassist |= (CSUM_IP6_TCP | CSUM_IP6_UDP); - if (adapter->hw.mac.type != e1000_82575) - ifp->if_hwassist |= CSUM_IP6_SCTP; - } -#endif - if (ifp->if_capenable & IFCAP_TSO) - ifp->if_hwassist |= CSUM_TSO; - - /* Clear bad data from Rx FIFOs */ - e1000_rx_fifo_flush_82575(&adapter->hw); - - /* Configure for OS presence */ - igb_init_manageability(adapter); - - /* Prepare transmit descriptors and buffers */ - igb_setup_transmit_structures(adapter); - igb_initialize_transmit_units(adapter); - - /* Setup Multicast table */ - igb_set_multi(adapter); - - /* - ** Figure out the desired mbuf pool - ** for doing jumbo/packetsplit - */ - if (adapter->max_frame_size <= 2048) - adapter->rx_mbuf_sz = MCLBYTES; - else if (adapter->max_frame_size <= 4096) - adapter->rx_mbuf_sz = MJUMPAGESIZE; - else - adapter->rx_mbuf_sz = MJUM9BYTES; - - /* Prepare receive descriptors and buffers */ - if (igb_setup_receive_structures(adapter)) { - device_printf(dev, "Could not setup receive structures\n"); - return; - } - igb_initialize_receive_units(adapter); - - /* Enable VLAN support */ - if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) - igb_setup_vlan_hw_support(adapter); - - /* Don't lose promiscuous settings */ - igb_set_promisc(adapter); - - ifp->if_drv_flags |= IFF_DRV_RUNNING; - ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; - - callout_reset(&adapter->timer, hz, igb_local_timer, adapter); - e1000_clear_hw_cntrs_base_generic(&adapter->hw); - - if (adapter->msix > 1) /* Set up queue routing */ - igb_configure_queues(adapter); - - /* this clears any pending interrupts */ - E1000_READ_REG(&adapter->hw, E1000_ICR); -#ifdef DEVICE_POLLING - /* - * Only enable interrupts if we are not polling, make sure - * they are off otherwise. - */ - if (ifp->if_capenable & IFCAP_POLLING) - igb_disable_intr(adapter); - else -#endif /* DEVICE_POLLING */ - { - igb_enable_intr(adapter); - E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC); - } - - /* Set Energy Efficient Ethernet */ - if (adapter->hw.phy.media_type == e1000_media_type_copper) { - if (adapter->hw.mac.type == e1000_i354) - e1000_set_eee_i354(&adapter->hw, TRUE, TRUE); - else - e1000_set_eee_i350(&adapter->hw, TRUE, TRUE); - } -} - -static void -igb_init(void *arg) -{ - struct adapter *adapter = arg; - - IGB_CORE_LOCK(adapter); - igb_init_locked(adapter); - IGB_CORE_UNLOCK(adapter); -} - - -static void -igb_handle_que(void *context, int pending) -{ - struct igb_queue *que = context; - struct adapter *adapter = que->adapter; - struct tx_ring *txr = que->txr; - struct ifnet *ifp = adapter->ifp; - - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - bool more; - - more = igb_rxeof(que, adapter->rx_process_limit, NULL); - - IGB_TX_LOCK(txr); - igb_txeof(txr); -#ifndef IGB_LEGACY_TX - /* Process the stack queue only if not depleted */ - if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) && - !drbr_empty(ifp, txr->br)) - igb_mq_start_locked(ifp, txr); -#else - if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) - igb_start_locked(txr, ifp); -#endif - IGB_TX_UNLOCK(txr); - /* Do we need another? */ - if (more) { - taskqueue_enqueue(que->tq, &que->que_task); - return; - } - } - -#ifdef DEVICE_POLLING - if (ifp->if_capenable & IFCAP_POLLING) - return; -#endif - /* Reenable this interrupt */ - if (que->eims) - E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims); - else - igb_enable_intr(adapter); -} - -/* Deal with link in a sleepable context */ -static void -igb_handle_link(void *context, int pending) -{ - struct adapter *adapter = context; - - IGB_CORE_LOCK(adapter); - igb_handle_link_locked(adapter); - IGB_CORE_UNLOCK(adapter); -} - -static void -igb_handle_link_locked(struct adapter *adapter) -{ - struct tx_ring *txr = adapter->tx_rings; - struct ifnet *ifp = adapter->ifp; - - IGB_CORE_LOCK_ASSERT(adapter); - adapter->hw.mac.get_link_status = 1; - igb_update_link_status(adapter); - if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) { - for (int i = 0; i < adapter->num_queues; i++, txr++) { - IGB_TX_LOCK(txr); -#ifndef IGB_LEGACY_TX - /* Process the stack queue only if not depleted */ - if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) && - !drbr_empty(ifp, txr->br)) - igb_mq_start_locked(ifp, txr); -#else - if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) - igb_start_locked(txr, ifp); -#endif - IGB_TX_UNLOCK(txr); - } - } -} - -/********************************************************************* - * - * MSI/Legacy Deferred - * Interrupt Service routine - * - *********************************************************************/ -static int -igb_irq_fast(void *arg) -{ - struct adapter *adapter = arg; - struct igb_queue *que = adapter->queues; - u32 reg_icr; - - - reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); - - /* Hot eject? */ - if (reg_icr == 0xffffffff) - return FILTER_STRAY; - - /* Definitely not our interrupt. */ - if (reg_icr == 0x0) - return FILTER_STRAY; - - if ((reg_icr & E1000_ICR_INT_ASSERTED) == 0) - return FILTER_STRAY; - - /* - * Mask interrupts until the taskqueue is finished running. This is - * cheap, just assume that it is needed. This also works around the - * MSI message reordering errata on certain systems. - */ - igb_disable_intr(adapter); - taskqueue_enqueue(que->tq, &que->que_task); - - /* Link status change */ - if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) - taskqueue_enqueue(que->tq, &adapter->link_task); - - if (reg_icr & E1000_ICR_RXO) - adapter->rx_overruns++; - return FILTER_HANDLED; -} - -#ifdef DEVICE_POLLING -#if __FreeBSD_version >= 800000 -#define POLL_RETURN_COUNT(a) (a) -static int -#else -#define POLL_RETURN_COUNT(a) -static void -#endif -igb_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) -{ - struct adapter *adapter = ifp->if_softc; - struct igb_queue *que; - struct tx_ring *txr; - u32 reg_icr, rx_done = 0; - u32 loop = IGB_MAX_LOOP; - bool more; - - IGB_CORE_LOCK(adapter); - if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { - IGB_CORE_UNLOCK(adapter); - return POLL_RETURN_COUNT(rx_done); - } - - if (cmd == POLL_AND_CHECK_STATUS) { - reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); - /* Link status change */ - if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) - igb_handle_link_locked(adapter); - - if (reg_icr & E1000_ICR_RXO) - adapter->rx_overruns++; - } - IGB_CORE_UNLOCK(adapter); - - for (int i = 0; i < adapter->num_queues; i++) { - que = &adapter->queues[i]; - txr = que->txr; - - igb_rxeof(que, count, &rx_done); - - IGB_TX_LOCK(txr); - do { - more = igb_txeof(txr); - } while (loop-- && more); -#ifndef IGB_LEGACY_TX - if (!drbr_empty(ifp, txr->br)) - igb_mq_start_locked(ifp, txr); -#else - if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) - igb_start_locked(txr, ifp); -#endif - IGB_TX_UNLOCK(txr); - } - - return POLL_RETURN_COUNT(rx_done); -} -#endif /* DEVICE_POLLING */ - -/********************************************************************* - * - * MSIX Que Interrupt Service routine - * - **********************************************************************/ -static void -igb_msix_que(void *arg) -{ - struct igb_queue *que = arg; - struct adapter *adapter = que->adapter; - struct ifnet *ifp = adapter->ifp; - struct tx_ring *txr = que->txr; - struct rx_ring *rxr = que->rxr; - u32 newitr = 0; - bool more_rx; - - /* Ignore spurious interrupts */ - if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) - return; - - E1000_WRITE_REG(&adapter->hw, E1000_EIMC, que->eims); - ++que->irqs; - - IGB_TX_LOCK(txr); - igb_txeof(txr); -#ifndef IGB_LEGACY_TX - /* Process the stack queue only if not depleted */ - if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) && - !drbr_empty(ifp, txr->br)) - igb_mq_start_locked(ifp, txr); -#else - if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) - igb_start_locked(txr, ifp); -#endif - IGB_TX_UNLOCK(txr); - - more_rx = igb_rxeof(que, adapter->rx_process_limit, NULL); - - if (adapter->enable_aim == FALSE) - goto no_calc; - /* - ** Do Adaptive Interrupt Moderation: - ** - Write out last calculated setting - ** - Calculate based on average size over - ** the last interval. - */ - if (que->eitr_setting) - E1000_WRITE_REG(&adapter->hw, - E1000_EITR(que->msix), que->eitr_setting); - - que->eitr_setting = 0; - - /* Idle, do nothing */ - if ((txr->bytes == 0) && (rxr->bytes == 0)) - goto no_calc; - - /* Used half Default if sub-gig */ - if (adapter->link_speed != 1000) - newitr = IGB_DEFAULT_ITR / 2; - else { - if ((txr->bytes) && (txr->packets)) - newitr = txr->bytes/txr->packets; - if ((rxr->bytes) && (rxr->packets)) - newitr = max(newitr, - (rxr->bytes / rxr->packets)); - newitr += 24; /* account for hardware frame, crc */ - /* set an upper boundary */ - newitr = min(newitr, 3000); - /* Be nice to the mid range */ - if ((newitr > 300) && (newitr < 1200)) - newitr = (newitr / 3); - else - newitr = (newitr / 2); - } - newitr &= 0x7FFC; /* Mask invalid bits */ - if (adapter->hw.mac.type == e1000_82575) - newitr |= newitr << 16; - else - newitr |= E1000_EITR_CNT_IGNR; - - /* save for next interrupt */ - que->eitr_setting = newitr; - - /* Reset state */ - txr->bytes = 0; - txr->packets = 0; - rxr->bytes = 0; - rxr->packets = 0; - -no_calc: - /* Schedule a clean task if needed*/ - if (more_rx) - taskqueue_enqueue(que->tq, &que->que_task); - else - /* Reenable this interrupt */ - E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims); - return; -} - - -/********************************************************************* - * - * MSIX Link Interrupt Service routine - * - **********************************************************************/ - -static void -igb_msix_link(void *arg) -{ - struct adapter *adapter = arg; - u32 icr; - - ++adapter->link_irq; - icr = E1000_READ_REG(&adapter->hw, E1000_ICR); - if (!(icr & E1000_ICR_LSC)) - goto spurious; - igb_handle_link(adapter, 0); - -spurious: - /* Rearm */ - E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC); - E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask); - return; -} - - -/********************************************************************* - * - * Media Ioctl callback - * - * This routine is called whenever the user queries the status of - * the interface using ifconfig. - * - **********************************************************************/ -static void -igb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) -{ - struct adapter *adapter = ifp->if_softc; - - INIT_DEBUGOUT("igb_media_status: begin"); - - IGB_CORE_LOCK(adapter); - igb_update_link_status(adapter); - - ifmr->ifm_status = IFM_AVALID; - ifmr->ifm_active = IFM_ETHER; - - if (!adapter->link_active) { - IGB_CORE_UNLOCK(adapter); - return; - } - - ifmr->ifm_status |= IFM_ACTIVE; - - switch (adapter->link_speed) { - case 10: - ifmr->ifm_active |= IFM_10_T; - break; - case 100: - /* - ** Support for 100Mb SFP - these are Fiber - ** but the media type appears as serdes - */ - if (adapter->hw.phy.media_type == - e1000_media_type_internal_serdes) - ifmr->ifm_active |= IFM_100_FX; - else - ifmr->ifm_active |= IFM_100_TX; - break; - case 1000: - ifmr->ifm_active |= IFM_1000_T; - break; - case 2500: - ifmr->ifm_active |= IFM_2500_SX; - break; - } - - if (adapter->link_duplex == FULL_DUPLEX) - ifmr->ifm_active |= IFM_FDX; - else - ifmr->ifm_active |= IFM_HDX; - - IGB_CORE_UNLOCK(adapter); -} - -/********************************************************************* - * - * Media Ioctl callback - * - * This routine is called when the user changes speed/duplex using - * media/mediopt option with ifconfig. - * - **********************************************************************/ -static int -igb_media_change(struct ifnet *ifp) -{ - struct adapter *adapter = ifp->if_softc; - struct ifmedia *ifm = &adapter->media; - - INIT_DEBUGOUT("igb_media_change: begin"); - - if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) - return (EINVAL); - - IGB_CORE_LOCK(adapter); - switch (IFM_SUBTYPE(ifm->ifm_media)) { - case IFM_AUTO: - adapter->hw.mac.autoneg = DO_AUTO_NEG; - adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; - break; - case IFM_1000_LX: - case IFM_1000_SX: - case IFM_1000_T: - adapter->hw.mac.autoneg = DO_AUTO_NEG; - adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; - break; - case IFM_100_TX: - adapter->hw.mac.autoneg = FALSE; - adapter->hw.phy.autoneg_advertised = 0; - if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) - adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; - else - adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; - break; - case IFM_10_T: - adapter->hw.mac.autoneg = FALSE; - adapter->hw.phy.autoneg_advertised = 0; - if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) - adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; - else - adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; - break; - default: - device_printf(adapter->dev, "Unsupported media type\n"); - } - - igb_init_locked(adapter); - IGB_CORE_UNLOCK(adapter); - - return (0); -} - - -/********************************************************************* - * - * This routine maps the mbufs to Advanced TX descriptors. - * - **********************************************************************/ -static int -igb_xmit(struct tx_ring *txr, struct mbuf **m_headp) -{ - struct adapter *adapter = txr->adapter; - u32 olinfo_status = 0, cmd_type_len; - int i, j, error, nsegs; - int first; - bool remap = TRUE; - struct mbuf *m_head; - bus_dma_segment_t segs[IGB_MAX_SCATTER]; - bus_dmamap_t map; - struct igb_tx_buf *txbuf; - union e1000_adv_tx_desc *txd = NULL; - - m_head = *m_headp; - - /* Basic descriptor defines */ - cmd_type_len = (E1000_ADVTXD_DTYP_DATA | - E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT); - - if (m_head->m_flags & M_VLANTAG) - cmd_type_len |= E1000_ADVTXD_DCMD_VLE; - - /* - * Important to capture the first descriptor - * used because it will contain the index of - * the one we tell the hardware to report back - */ - first = txr->next_avail_desc; - txbuf = &txr->tx_buffers[first]; - map = txbuf->map; - - /* - * Map the packet for DMA. - */ -retry: - error = bus_dmamap_load_mbuf_sg(txr->txtag, map, - *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); - - if (__predict_false(error)) { - struct mbuf *m; - - switch (error) { - case EFBIG: - /* Try it again? - one try */ - if (remap == TRUE) { - remap = FALSE; - m = m_collapse(*m_headp, M_NOWAIT, - IGB_MAX_SCATTER); - if (m == NULL) { - adapter->mbuf_defrag_failed++; - m_freem(*m_headp); - *m_headp = NULL; - return (ENOBUFS); - } - *m_headp = m; - goto retry; - } else - return (error); - default: - txr->no_tx_dma_setup++; - m_freem(*m_headp); - *m_headp = NULL; - return (error); - } - } - - /* Make certain there are enough descriptors */ - if (txr->tx_avail < (nsegs + 2)) { - txr->no_desc_avail++; - bus_dmamap_unload(txr->txtag, map); - return (ENOBUFS); - } - m_head = *m_headp; - - /* - ** Set up the appropriate offload context - ** this will consume the first descriptor - */ - error = igb_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status); - if (__predict_false(error)) { - m_freem(*m_headp); - *m_headp = NULL; - return (error); - } - - /* 82575 needs the queue index added */ - if (adapter->hw.mac.type == e1000_82575) - olinfo_status |= txr->me << 4; - - i = txr->next_avail_desc; - for (j = 0; j < nsegs; j++) { - bus_size_t seglen; - bus_addr_t segaddr; - - txbuf = &txr->tx_buffers[i]; - txd = &txr->tx_base[i]; - seglen = segs[j].ds_len; - segaddr = htole64(segs[j].ds_addr); - - txd->read.buffer_addr = segaddr; - txd->read.cmd_type_len = htole32(E1000_TXD_CMD_IFCS | - cmd_type_len | seglen); - txd->read.olinfo_status = htole32(olinfo_status); - - if (++i == txr->num_desc) - i = 0; - } - - txd->read.cmd_type_len |= - htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS); - txr->tx_avail -= nsegs; - txr->next_avail_desc = i; - - txbuf->m_head = m_head; - /* - ** Here we swap the map so the last descriptor, - ** which gets the completion interrupt has the - ** real map, and the first descriptor gets the - ** unused map from this descriptor. - */ - txr->tx_buffers[first].map = txbuf->map; - txbuf->map = map; - bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); - - /* Set the EOP descriptor that will be marked done */ - txbuf = &txr->tx_buffers[first]; - txbuf->eop = txd; - - bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - /* - * Advance the Transmit Descriptor Tail (Tdt), this tells the - * hardware that this frame is available to transmit. - */ - ++txr->total_packets; - E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i); - - return (0); -} -static void -igb_set_promisc(struct adapter *adapter) -{ - struct ifnet *ifp = adapter->ifp; - struct e1000_hw *hw = &adapter->hw; - u32 reg; - - if (adapter->vf_ifp) { - e1000_promisc_set_vf(hw, e1000_promisc_enabled); - return; - } - - reg = E1000_READ_REG(hw, E1000_RCTL); - if (ifp->if_flags & IFF_PROMISC) { - reg |= (E1000_RCTL_UPE | E1000_RCTL_MPE); - E1000_WRITE_REG(hw, E1000_RCTL, reg); - } else if (ifp->if_flags & IFF_ALLMULTI) { - reg |= E1000_RCTL_MPE; - reg &= ~E1000_RCTL_UPE; - E1000_WRITE_REG(hw, E1000_RCTL, reg); - } -} - -static void -igb_disable_promisc(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - struct ifnet *ifp = adapter->ifp; - u32 reg; - int mcnt = 0; - - if (adapter->vf_ifp) { - e1000_promisc_set_vf(hw, e1000_promisc_disabled); - return; - } - reg = E1000_READ_REG(hw, E1000_RCTL); - reg &= (~E1000_RCTL_UPE); - if (ifp->if_flags & IFF_ALLMULTI) - mcnt = MAX_NUM_MULTICAST_ADDRESSES; - else { - struct ifmultiaddr *ifma; -#if __FreeBSD_version < 800000 - IF_ADDR_LOCK(ifp); -#else - if_maddr_rlock(ifp); -#endif - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { - if (ifma->ifma_addr->sa_family != AF_LINK) - continue; - if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) - break; - mcnt++; - } -#if __FreeBSD_version < 800000 - IF_ADDR_UNLOCK(ifp); -#else - if_maddr_runlock(ifp); -#endif - } - /* Don't disable if in MAX groups */ - if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) - reg &= (~E1000_RCTL_MPE); - E1000_WRITE_REG(hw, E1000_RCTL, reg); -} - - -/********************************************************************* - * Multicast Update - * - * This routine is called whenever multicast address list is updated. - * - **********************************************************************/ - -static void -igb_set_multi(struct adapter *adapter) -{ - struct ifnet *ifp = adapter->ifp; - struct ifmultiaddr *ifma; - u32 reg_rctl = 0; - u8 *mta; - - int mcnt = 0; - - IOCTL_DEBUGOUT("igb_set_multi: begin"); - - mta = adapter->mta; - bzero(mta, sizeof(uint8_t) * ETH_ADDR_LEN * - MAX_NUM_MULTICAST_ADDRESSES); - -#if __FreeBSD_version < 800000 - IF_ADDR_LOCK(ifp); -#else - if_maddr_rlock(ifp); -#endif - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { - if (ifma->ifma_addr->sa_family != AF_LINK) - continue; - - if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) - break; - - bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), - &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN); - mcnt++; - } -#if __FreeBSD_version < 800000 - IF_ADDR_UNLOCK(ifp); -#else - if_maddr_runlock(ifp); -#endif - - if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) { - reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); - reg_rctl |= E1000_RCTL_MPE; - E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); - } else - e1000_update_mc_addr_list(&adapter->hw, mta, mcnt); -} - - -/********************************************************************* - * Timer routine: - * This routine checks for link status, - * updates statistics, and does the watchdog. - * - **********************************************************************/ - -static void -igb_local_timer(void *arg) -{ - struct adapter *adapter = arg; - device_t dev = adapter->dev; - struct ifnet *ifp = adapter->ifp; - struct tx_ring *txr = adapter->tx_rings; - struct igb_queue *que = adapter->queues; - int hung = 0, busy = 0; - - - IGB_CORE_LOCK_ASSERT(adapter); - - igb_update_link_status(adapter); - igb_update_stats_counters(adapter); - - /* - ** Check the TX queues status - ** - central locked handling of OACTIVE - ** - watchdog only if all queues show hung - */ - for (int i = 0; i < adapter->num_queues; i++, que++, txr++) { - if ((txr->queue_status & IGB_QUEUE_HUNG) && - (adapter->pause_frames == 0)) - ++hung; - if (txr->queue_status & IGB_QUEUE_DEPLETED) - ++busy; - if ((txr->queue_status & IGB_QUEUE_IDLE) == 0) - taskqueue_enqueue(que->tq, &que->que_task); - } - if (hung == adapter->num_queues) - goto timeout; - if (busy == adapter->num_queues) - ifp->if_drv_flags |= IFF_DRV_OACTIVE; - else if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) && - (busy < adapter->num_queues)) - ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; - - adapter->pause_frames = 0; - callout_reset(&adapter->timer, hz, igb_local_timer, adapter); -#ifndef DEVICE_POLLING - /* Schedule all queue interrupts - deadlock protection */ - E1000_WRITE_REG(&adapter->hw, E1000_EICS, adapter->que_mask); -#endif - return; - -timeout: - device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); - device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, - E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)), - E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me))); - device_printf(dev,"TX(%d) desc avail = %d," - "Next TX to Clean = %d\n", - txr->me, txr->tx_avail, txr->next_to_clean); - adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; - adapter->watchdog_events++; - igb_init_locked(adapter); -} - -static void -igb_update_link_status(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - struct e1000_fc_info *fc = &hw->fc; - struct ifnet *ifp = adapter->ifp; - device_t dev = adapter->dev; - struct tx_ring *txr = adapter->tx_rings; - u32 link_check, thstat, ctrl; - char *flowctl = NULL; - - link_check = thstat = ctrl = 0; - - /* Get the cached link value or read for real */ - switch (hw->phy.media_type) { - case e1000_media_type_copper: - if (hw->mac.get_link_status) { - /* Do the work to read phy */ - e1000_check_for_link(hw); - link_check = !hw->mac.get_link_status; - } else - link_check = TRUE; - break; - case e1000_media_type_fiber: - e1000_check_for_link(hw); - link_check = (E1000_READ_REG(hw, E1000_STATUS) & - E1000_STATUS_LU); - break; - case e1000_media_type_internal_serdes: - e1000_check_for_link(hw); - link_check = adapter->hw.mac.serdes_has_link; - break; - /* VF device is type_unknown */ - case e1000_media_type_unknown: - e1000_check_for_link(hw); - link_check = !hw->mac.get_link_status; - /* Fall thru */ - default: - break; - } - - /* Check for thermal downshift or shutdown */ - if (hw->mac.type == e1000_i350) { - thstat = E1000_READ_REG(hw, E1000_THSTAT); - ctrl = E1000_READ_REG(hw, E1000_CTRL_EXT); - } - - /* Get the flow control for display */ - switch (fc->current_mode) { - case e1000_fc_rx_pause: - flowctl = "RX"; - break; - case e1000_fc_tx_pause: - flowctl = "TX"; - break; - case e1000_fc_full: - flowctl = "Full"; - break; - case e1000_fc_none: - default: - flowctl = "None"; - break; - } - - /* Now we check if a transition has happened */ - if (link_check && (adapter->link_active == 0)) { - e1000_get_speed_and_duplex(&adapter->hw, - &adapter->link_speed, &adapter->link_duplex); - if (bootverbose) - device_printf(dev, "Link is up %d Mbps %s," - " Flow Control: %s\n", - adapter->link_speed, - ((adapter->link_duplex == FULL_DUPLEX) ? - "Full Duplex" : "Half Duplex"), flowctl); - adapter->link_active = 1; - ifp->if_baudrate = adapter->link_speed * 1000000; - if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) && - (thstat & E1000_THSTAT_LINK_THROTTLE)) - device_printf(dev, "Link: thermal downshift\n"); - /* Delay Link Up for Phy update */ - if (((hw->mac.type == e1000_i210) || - (hw->mac.type == e1000_i211)) && - (hw->phy.id == I210_I_PHY_ID)) - msec_delay(I210_LINK_DELAY); - /* Reset if the media type changed. */ - if (hw->dev_spec._82575.media_changed) { - hw->dev_spec._82575.media_changed = false; - adapter->flags |= IGB_MEDIA_RESET; - igb_reset(adapter); - } - /* This can sleep */ - if_link_state_change(ifp, LINK_STATE_UP); - } else if (!link_check && (adapter->link_active == 1)) { - ifp->if_baudrate = adapter->link_speed = 0; - adapter->link_duplex = 0; - if (bootverbose) - device_printf(dev, "Link is Down\n"); - if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) && - (thstat & E1000_THSTAT_PWR_DOWN)) - device_printf(dev, "Link: thermal shutdown\n"); - adapter->link_active = 0; - /* This can sleep */ - if_link_state_change(ifp, LINK_STATE_DOWN); - /* Reset queue state */ - for (int i = 0; i < adapter->num_queues; i++, txr++) - txr->queue_status = IGB_QUEUE_IDLE; - } -} - -/********************************************************************* - * - * This routine disables all traffic on the adapter by issuing a - * global reset on the MAC and deallocates TX/RX buffers. - * - **********************************************************************/ - -static void -igb_stop(void *arg) -{ - struct adapter *adapter = arg; - struct ifnet *ifp = adapter->ifp; - struct tx_ring *txr = adapter->tx_rings; - - IGB_CORE_LOCK_ASSERT(adapter); - - INIT_DEBUGOUT("igb_stop: begin"); - - igb_disable_intr(adapter); - - callout_stop(&adapter->timer); - - /* Tell the stack that the interface is no longer active */ - ifp->if_drv_flags &= ~IFF_DRV_RUNNING; - ifp->if_drv_flags |= IFF_DRV_OACTIVE; - - /* Disarm watchdog timer. */ - for (int i = 0; i < adapter->num_queues; i++, txr++) { - IGB_TX_LOCK(txr); - txr->queue_status = IGB_QUEUE_IDLE; - IGB_TX_UNLOCK(txr); - } - - e1000_reset_hw(&adapter->hw); - E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); - - e1000_led_off(&adapter->hw); - e1000_cleanup_led(&adapter->hw); -} - - -/********************************************************************* - * - * Determine hardware revision. - * - **********************************************************************/ -static void -igb_identify_hardware(struct adapter *adapter) -{ - device_t dev = adapter->dev; - - /* Make sure our PCI config space has the necessary stuff set */ - pci_enable_busmaster(dev); - adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); - - /* Save off the information about this board */ - adapter->hw.vendor_id = pci_get_vendor(dev); - adapter->hw.device_id = pci_get_device(dev); - adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); - adapter->hw.subsystem_vendor_id = - pci_read_config(dev, PCIR_SUBVEND_0, 2); - adapter->hw.subsystem_device_id = - pci_read_config(dev, PCIR_SUBDEV_0, 2); - - /* Set MAC type early for PCI setup */ - e1000_set_mac_type(&adapter->hw); - - /* Are we a VF device? */ - if ((adapter->hw.mac.type == e1000_vfadapt) || - (adapter->hw.mac.type == e1000_vfadapt_i350)) - adapter->vf_ifp = 1; - else - adapter->vf_ifp = 0; -} - -static int -igb_allocate_pci_resources(struct adapter *adapter) -{ - device_t dev = adapter->dev; - int rid; - - rid = PCIR_BAR(0); - adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, - &rid, RF_ACTIVE); - if (adapter->pci_mem == NULL) { - device_printf(dev, "Unable to allocate bus resource: memory\n"); - return (ENXIO); - } - adapter->osdep.mem_bus_space_tag = - rman_get_bustag(adapter->pci_mem); - adapter->osdep.mem_bus_space_handle = - rman_get_bushandle(adapter->pci_mem); - adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; - - adapter->num_queues = 1; /* Defaults for Legacy or MSI */ - - /* This will setup either MSI/X or MSI */ - adapter->msix = igb_setup_msix(adapter); - adapter->hw.back = &adapter->osdep; - - return (0); -} - -/********************************************************************* - * - * Setup the Legacy or MSI Interrupt handler - * - **********************************************************************/ -static int -igb_allocate_legacy(struct adapter *adapter) -{ - device_t dev = adapter->dev; - struct igb_queue *que = adapter->queues; -#ifndef IGB_LEGACY_TX - struct tx_ring *txr = adapter->tx_rings; -#endif - int error, rid = 0; - - /* Turn off all interrupts */ - E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); - - /* MSI RID is 1 */ - if (adapter->msix == 1) - rid = 1; - - /* We allocate a single interrupt resource */ - adapter->res = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); - if (adapter->res == NULL) { - device_printf(dev, "Unable to allocate bus resource: " - "interrupt\n"); - return (ENXIO); - } - -#ifndef IGB_LEGACY_TX - TASK_INIT(&txr->txq_task, 0, igb_deferred_mq_start, txr); -#endif - - /* - * Try allocating a fast interrupt and the associated deferred - * processing contexts. - */ - TASK_INIT(&que->que_task, 0, igb_handle_que, que); - /* Make tasklet for deferred link handling */ - TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter); - que->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT, - taskqueue_thread_enqueue, &que->tq); - taskqueue_start_threads(&que->tq, 1, PI_NET, "%s taskq", - device_get_nameunit(adapter->dev)); - if ((error = bus_setup_intr(dev, adapter->res, - INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast, NULL, - adapter, &adapter->tag)) != 0) { - device_printf(dev, "Failed to register fast interrupt " - "handler: %d\n", error); - taskqueue_free(que->tq); - que->tq = NULL; - return (error); - } - - return (0); -} - - -/********************************************************************* - * - * Setup the MSIX Queue Interrupt handlers: - * - **********************************************************************/ -static int -igb_allocate_msix(struct adapter *adapter) -{ - device_t dev = adapter->dev; - struct igb_queue *que = adapter->queues; - int error, rid, vector = 0; - int cpu_id = 0; -#ifdef RSS - cpuset_t cpu_mask; -#endif - - /* Be sure to start with all interrupts disabled */ - E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0); - E1000_WRITE_FLUSH(&adapter->hw); - -#ifdef RSS - /* - * If we're doing RSS, the number of queues needs to - * match the number of RSS buckets that are configured. - * - * + If there's more queues than RSS buckets, we'll end - * up with queues that get no traffic. - * - * + If there's more RSS buckets than queues, we'll end - * up having multiple RSS buckets map to the same queue, - * so there'll be some contention. - */ - if (adapter->num_queues != rss_getnumbuckets()) { - device_printf(dev, - "%s: number of queues (%d) != number of RSS buckets (%d)" - "; performance will be impacted.\n", - __func__, - adapter->num_queues, - rss_getnumbuckets()); - } -#endif - - for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { - rid = vector +1; - que->res = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); - if (que->res == NULL) { - device_printf(dev, - "Unable to allocate bus resource: " - "MSIX Queue Interrupt\n"); - return (ENXIO); - } - error = bus_setup_intr(dev, que->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, - igb_msix_que, que, &que->tag); - if (error) { - que->res = NULL; - device_printf(dev, "Failed to register Queue handler"); - return (error); - } -#if __FreeBSD_version >= 800504 - bus_describe_intr(dev, que->res, que->tag, "que %d", i); -#endif - que->msix = vector; - if (adapter->hw.mac.type == e1000_82575) - que->eims = E1000_EICR_TX_QUEUE0 << i; - else - que->eims = 1 << vector; - -#ifdef RSS - /* - * The queue ID is used as the RSS layer bucket ID. - * We look up the queue ID -> RSS CPU ID and select - * that. - */ - cpu_id = rss_getcpu(i % rss_getnumbuckets()); -#else - /* - * Bind the msix vector, and thus the - * rings to the corresponding cpu. - * - * This just happens to match the default RSS round-robin - * bucket -> queue -> CPU allocation. - */ - if (adapter->num_queues > 1) { - if (igb_last_bind_cpu < 0) - igb_last_bind_cpu = CPU_FIRST(); - cpu_id = igb_last_bind_cpu; - } -#endif - - if (adapter->num_queues > 1) { - bus_bind_intr(dev, que->res, cpu_id); -#ifdef RSS - device_printf(dev, - "Bound queue %d to RSS bucket %d\n", - i, cpu_id); -#else - device_printf(dev, - "Bound queue %d to cpu %d\n", - i, cpu_id); -#endif - } - -#ifndef IGB_LEGACY_TX - TASK_INIT(&que->txr->txq_task, 0, igb_deferred_mq_start, - que->txr); -#endif - /* Make tasklet for deferred handling */ - TASK_INIT(&que->que_task, 0, igb_handle_que, que); - que->tq = taskqueue_create("igb_que", M_NOWAIT, - taskqueue_thread_enqueue, &que->tq); - if (adapter->num_queues > 1) { - /* - * Only pin the taskqueue thread to a CPU if - * RSS is in use. - * - * This again just happens to match the default RSS - * round-robin bucket -> queue -> CPU allocation. - */ -#ifdef RSS - CPU_SETOF(cpu_id, &cpu_mask); - taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET, - &cpu_mask, - "%s que (bucket %d)", - device_get_nameunit(adapter->dev), - cpu_id); -#else - taskqueue_start_threads(&que->tq, 1, PI_NET, - "%s que (qid %d)", - device_get_nameunit(adapter->dev), - cpu_id); -#endif - } else { - taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que", - device_get_nameunit(adapter->dev)); - } - - /* Finally update the last bound CPU id */ - if (adapter->num_queues > 1) - igb_last_bind_cpu = CPU_NEXT(igb_last_bind_cpu); - } - - /* And Link */ - rid = vector + 1; - adapter->res = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); - if (adapter->res == NULL) { - device_printf(dev, - "Unable to allocate bus resource: " - "MSIX Link Interrupt\n"); - return (ENXIO); - } - if ((error = bus_setup_intr(dev, adapter->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, - igb_msix_link, adapter, &adapter->tag)) != 0) { - device_printf(dev, "Failed to register Link handler"); - return (error); - } -#if __FreeBSD_version >= 800504 - bus_describe_intr(dev, adapter->res, adapter->tag, "link"); -#endif - adapter->linkvec = vector; - - return (0); -} - - -static void -igb_configure_queues(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - struct igb_queue *que; - u32 tmp, ivar = 0, newitr = 0; - - /* First turn on RSS capability */ - if (adapter->hw.mac.type != e1000_82575) - E1000_WRITE_REG(hw, E1000_GPIE, - E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME | - E1000_GPIE_PBA | E1000_GPIE_NSICR); - - /* Turn on MSIX */ - switch (adapter->hw.mac.type) { - case e1000_82580: - case e1000_i350: - case e1000_i354: - case e1000_i210: - case e1000_i211: - case e1000_vfadapt: - case e1000_vfadapt_i350: - /* RX entries */ - for (int i = 0; i < adapter->num_queues; i++) { - u32 index = i >> 1; - ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); - que = &adapter->queues[i]; - if (i & 1) { - ivar &= 0xFF00FFFF; - ivar |= (que->msix | E1000_IVAR_VALID) << 16; - } else { - ivar &= 0xFFFFFF00; - ivar |= que->msix | E1000_IVAR_VALID; - } - E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); - } - /* TX entries */ - for (int i = 0; i < adapter->num_queues; i++) { - u32 index = i >> 1; - ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); - que = &adapter->queues[i]; - if (i & 1) { - ivar &= 0x00FFFFFF; - ivar |= (que->msix | E1000_IVAR_VALID) << 24; - } else { - ivar &= 0xFFFF00FF; - ivar |= (que->msix | E1000_IVAR_VALID) << 8; - } - E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); - adapter->que_mask |= que->eims; - } - - /* And for the link interrupt */ - ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8; - adapter->link_mask = 1 << adapter->linkvec; - E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); - break; - case e1000_82576: - /* RX entries */ - for (int i = 0; i < adapter->num_queues; i++) { - u32 index = i & 0x7; /* Each IVAR has two entries */ - ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); - que = &adapter->queues[i]; - if (i < 8) { - ivar &= 0xFFFFFF00; - ivar |= que->msix | E1000_IVAR_VALID; - } else { - ivar &= 0xFF00FFFF; - ivar |= (que->msix | E1000_IVAR_VALID) << 16; - } - E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); - adapter->que_mask |= que->eims; - } - /* TX entries */ - for (int i = 0; i < adapter->num_queues; i++) { - u32 index = i & 0x7; /* Each IVAR has two entries */ - ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); - que = &adapter->queues[i]; - if (i < 8) { - ivar &= 0xFFFF00FF; - ivar |= (que->msix | E1000_IVAR_VALID) << 8; - } else { - ivar &= 0x00FFFFFF; - ivar |= (que->msix | E1000_IVAR_VALID) << 24; - } - E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); - adapter->que_mask |= que->eims; - } - - /* And for the link interrupt */ - ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8; - adapter->link_mask = 1 << adapter->linkvec; - E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); - break; - - case e1000_82575: - /* enable MSI-X support*/ - tmp = E1000_READ_REG(hw, E1000_CTRL_EXT); - tmp |= E1000_CTRL_EXT_PBA_CLR; - /* Auto-Mask interrupts upon ICR read. */ - tmp |= E1000_CTRL_EXT_EIAME; - tmp |= E1000_CTRL_EXT_IRCA; - E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp); - - /* Queues */ - for (int i = 0; i < adapter->num_queues; i++) { - que = &adapter->queues[i]; - tmp = E1000_EICR_RX_QUEUE0 << i; - tmp |= E1000_EICR_TX_QUEUE0 << i; - que->eims = tmp; - E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), - i, que->eims); - adapter->que_mask |= que->eims; - } - - /* Link */ - E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec), - E1000_EIMS_OTHER); - adapter->link_mask |= E1000_EIMS_OTHER; - default: - break; - } - - /* Set the starting interrupt rate */ - if (igb_max_interrupt_rate > 0) - newitr = (4000000 / igb_max_interrupt_rate) & 0x7FFC; - - if (hw->mac.type == e1000_82575) - newitr |= newitr << 16; - else - newitr |= E1000_EITR_CNT_IGNR; - - for (int i = 0; i < adapter->num_queues; i++) { - que = &adapter->queues[i]; - E1000_WRITE_REG(hw, E1000_EITR(que->msix), newitr); - } - - return; -} - - -static void -igb_free_pci_resources(struct adapter *adapter) -{ - struct igb_queue *que = adapter->queues; - device_t dev = adapter->dev; - int rid; - - /* - ** There is a slight possibility of a failure mode - ** in attach that will result in entering this function - ** before interrupt resources have been initialized, and - ** in that case we do not want to execute the loops below - ** We can detect this reliably by the state of the adapter - ** res pointer. - */ - if (adapter->res == NULL) - goto mem; - - /* - * First release all the interrupt resources: - */ - for (int i = 0; i < adapter->num_queues; i++, que++) { - rid = que->msix + 1; - if (que->tag != NULL) { - bus_teardown_intr(dev, que->res, que->tag); - que->tag = NULL; - } - if (que->res != NULL) - bus_release_resource(dev, - SYS_RES_IRQ, rid, que->res); - } - - /* Clean the Legacy or Link interrupt last */ - if (adapter->linkvec) /* we are doing MSIX */ - rid = adapter->linkvec + 1; - else - (adapter->msix != 0) ? (rid = 1):(rid = 0); - - que = adapter->queues; - if (adapter->tag != NULL) { - taskqueue_drain(que->tq, &adapter->link_task); - bus_teardown_intr(dev, adapter->res, adapter->tag); - adapter->tag = NULL; - } - if (adapter->res != NULL) - bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res); - - for (int i = 0; i < adapter->num_queues; i++, que++) { - if (que->tq != NULL) { -#ifndef IGB_LEGACY_TX - taskqueue_drain(que->tq, &que->txr->txq_task); -#endif - taskqueue_drain(que->tq, &que->que_task); - taskqueue_free(que->tq); - } - } -mem: - if (adapter->msix) - pci_release_msi(dev); - - if (adapter->msix_mem != NULL) - bus_release_resource(dev, SYS_RES_MEMORY, - adapter->memrid, adapter->msix_mem); - - if (adapter->pci_mem != NULL) - bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(0), adapter->pci_mem); - -} - -/* - * Setup Either MSI/X or MSI - */ -static int -igb_setup_msix(struct adapter *adapter) -{ - device_t dev = adapter->dev; - int bar, want, queues, msgs, maxqueues; - - /* tuneable override */ - if (igb_enable_msix == 0) - goto msi; - - /* First try MSI/X */ - msgs = pci_msix_count(dev); - if (msgs == 0) - goto msi; - /* - ** Some new devices, as with ixgbe, now may - ** use a different BAR, so we need to keep - ** track of which is used. - */ - adapter->memrid = PCIR_BAR(IGB_MSIX_BAR); - bar = pci_read_config(dev, adapter->memrid, 4); - if (bar == 0) /* use next bar */ - adapter->memrid += 4; - adapter->msix_mem = bus_alloc_resource_any(dev, - SYS_RES_MEMORY, &adapter->memrid, RF_ACTIVE); - if (adapter->msix_mem == NULL) { - /* May not be enabled */ - device_printf(adapter->dev, - "Unable to map MSIX table \n"); - goto msi; - } - - queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus; - - /* Override via tuneable */ - if (igb_num_queues != 0) - queues = igb_num_queues; - -#ifdef RSS - /* If we're doing RSS, clamp at the number of RSS buckets */ - if (queues > rss_getnumbuckets()) - queues = rss_getnumbuckets(); -#endif - - - /* Sanity check based on HW */ - switch (adapter->hw.mac.type) { - case e1000_82575: - maxqueues = 4; - break; - case e1000_82576: - case e1000_82580: - case e1000_i350: - case e1000_i354: - maxqueues = 8; - break; - case e1000_i210: - maxqueues = 4; - break; - case e1000_i211: - maxqueues = 2; - break; - default: /* VF interfaces */ - maxqueues = 1; - break; - } - - /* Final clamp on the actual hardware capability */ - if (queues > maxqueues) - queues = maxqueues; - - /* - ** One vector (RX/TX pair) per queue - ** plus an additional for Link interrupt - */ - want = queues + 1; - if (msgs >= want) - msgs = want; - else { - device_printf(adapter->dev, - "MSIX Configuration Problem, " - "%d vectors configured, but %d queues wanted!\n", - msgs, want); - goto msi; - } - if ((pci_alloc_msix(dev, &msgs) == 0) && (msgs == want)) { - device_printf(adapter->dev, - "Using MSIX interrupts with %d vectors\n", msgs); - adapter->num_queues = queues; - return (msgs); - } - /* - ** If MSIX alloc failed or provided us with - ** less than needed, free and fall through to MSI - */ - pci_release_msi(dev); - -msi: - if (adapter->msix_mem != NULL) { - bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem); - adapter->msix_mem = NULL; - } - msgs = 1; - if (pci_alloc_msi(dev, &msgs) == 0) { - device_printf(adapter->dev," Using an MSI interrupt\n"); - return (msgs); - } - device_printf(adapter->dev," Using a Legacy interrupt\n"); - return (0); -} - -/********************************************************************* - * - * Initialize the DMA Coalescing feature - * - **********************************************************************/ -static void -igb_init_dmac(struct adapter *adapter, u32 pba) -{ - device_t dev = adapter->dev; - struct e1000_hw *hw = &adapter->hw; - u32 dmac, reg = ~E1000_DMACR_DMAC_EN; - u16 hwm; - - if (hw->mac.type == e1000_i211) - return; - - if (hw->mac.type > e1000_82580) { - - if (adapter->dmac == 0) { /* Disabling it */ - E1000_WRITE_REG(hw, E1000_DMACR, reg); - return; - } else - device_printf(dev, "DMA Coalescing enabled\n"); - - /* Set starting threshold */ - E1000_WRITE_REG(hw, E1000_DMCTXTH, 0); - - hwm = 64 * pba - adapter->max_frame_size / 16; - if (hwm < 64 * (pba - 6)) - hwm = 64 * (pba - 6); - reg = E1000_READ_REG(hw, E1000_FCRTC); - reg &= ~E1000_FCRTC_RTH_COAL_MASK; - reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT) - & E1000_FCRTC_RTH_COAL_MASK); - E1000_WRITE_REG(hw, E1000_FCRTC, reg); - - - dmac = pba - adapter->max_frame_size / 512; - if (dmac < pba - 10) - dmac = pba - 10; - reg = E1000_READ_REG(hw, E1000_DMACR); - reg &= ~E1000_DMACR_DMACTHR_MASK; - reg = ((dmac << E1000_DMACR_DMACTHR_SHIFT) - & E1000_DMACR_DMACTHR_MASK); - - /* transition to L0x or L1 if available..*/ - reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK); - - /* Check if status is 2.5Gb backplane connection - * before configuration of watchdog timer, which is - * in msec values in 12.8usec intervals - * watchdog timer= msec values in 32usec intervals - * for non 2.5Gb connection - */ - if (hw->mac.type == e1000_i354) { - int status = E1000_READ_REG(hw, E1000_STATUS); - if ((status & E1000_STATUS_2P5_SKU) && - (!(status & E1000_STATUS_2P5_SKU_OVER))) - reg |= ((adapter->dmac * 5) >> 6); - else - reg |= (adapter->dmac >> 5); - } else { - reg |= (adapter->dmac >> 5); - } - - E1000_WRITE_REG(hw, E1000_DMACR, reg); - - E1000_WRITE_REG(hw, E1000_DMCRTRH, 0); - - /* Set the interval before transition */ - reg = E1000_READ_REG(hw, E1000_DMCTLX); - if (hw->mac.type == e1000_i350) - reg |= IGB_DMCTLX_DCFLUSH_DIS; - /* - ** in 2.5Gb connection, TTLX unit is 0.4 usec - ** which is 0x4*2 = 0xA. But delay is still 4 usec - */ - if (hw->mac.type == e1000_i354) { - int status = E1000_READ_REG(hw, E1000_STATUS); - if ((status & E1000_STATUS_2P5_SKU) && - (!(status & E1000_STATUS_2P5_SKU_OVER))) - reg |= 0xA; - else - reg |= 0x4; - } else { - reg |= 0x4; - } - - E1000_WRITE_REG(hw, E1000_DMCTLX, reg); - - /* free space in tx packet buffer to wake from DMA coal */ - E1000_WRITE_REG(hw, E1000_DMCTXTH, (IGB_TXPBSIZE - - (2 * adapter->max_frame_size)) >> 6); - - /* make low power state decision controlled by DMA coal */ - reg = E1000_READ_REG(hw, E1000_PCIEMISC); - reg &= ~E1000_PCIEMISC_LX_DECISION; - E1000_WRITE_REG(hw, E1000_PCIEMISC, reg); - - } else if (hw->mac.type == e1000_82580) { - u32 reg = E1000_READ_REG(hw, E1000_PCIEMISC); - E1000_WRITE_REG(hw, E1000_PCIEMISC, - reg & ~E1000_PCIEMISC_LX_DECISION); - E1000_WRITE_REG(hw, E1000_DMACR, 0); - } -} - - -/********************************************************************* - * - * Set up an fresh starting state - * - **********************************************************************/ -static void -igb_reset(struct adapter *adapter) -{ - device_t dev = adapter->dev; - struct e1000_hw *hw = &adapter->hw; - struct e1000_fc_info *fc = &hw->fc; - struct ifnet *ifp = adapter->ifp; - u32 pba = 0; - u16 hwm; - - INIT_DEBUGOUT("igb_reset: begin"); - - /* Let the firmware know the OS is in control */ - igb_get_hw_control(adapter); - - /* - * Packet Buffer Allocation (PBA) - * Writing PBA sets the receive portion of the buffer - * the remainder is used for the transmit buffer. - */ - switch (hw->mac.type) { - case e1000_82575: - pba = E1000_PBA_32K; - break; - case e1000_82576: - case e1000_vfadapt: - pba = E1000_READ_REG(hw, E1000_RXPBS); - pba &= E1000_RXPBS_SIZE_MASK_82576; - break; - case e1000_82580: - case e1000_i350: - case e1000_i354: - case e1000_vfadapt_i350: - pba = E1000_READ_REG(hw, E1000_RXPBS); - pba = e1000_rxpbs_adjust_82580(pba); - break; - case e1000_i210: - case e1000_i211: - pba = E1000_PBA_34K; - default: - break; - } - - /* Special needs in case of Jumbo frames */ - if ((hw->mac.type == e1000_82575) && (ifp->if_mtu > ETHERMTU)) { - u32 tx_space, min_tx, min_rx; - pba = E1000_READ_REG(hw, E1000_PBA); - tx_space = pba >> 16; - pba &= 0xffff; - min_tx = (adapter->max_frame_size + - sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2; - min_tx = roundup2(min_tx, 1024); - min_tx >>= 10; - min_rx = adapter->max_frame_size; - min_rx = roundup2(min_rx, 1024); - min_rx >>= 10; - if (tx_space < min_tx && - ((min_tx - tx_space) < pba)) { - pba = pba - (min_tx - tx_space); - /* - * if short on rx space, rx wins - * and must trump tx adjustment - */ - if (pba < min_rx) - pba = min_rx; - } - E1000_WRITE_REG(hw, E1000_PBA, pba); - } - - INIT_DEBUGOUT1("igb_init: pba=%dK",pba); - - /* - * These parameters control the automatic generation (Tx) and - * response (Rx) to Ethernet PAUSE frames. - * - High water mark should allow for at least two frames to be - * received after sending an XOFF. - * - Low water mark works best when it is very near the high water mark. - * This allows the receiver to restart by sending XON when it has - * drained a bit. - */ - hwm = min(((pba << 10) * 9 / 10), - ((pba << 10) - 2 * adapter->max_frame_size)); - - if (hw->mac.type < e1000_82576) { - fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */ - fc->low_water = fc->high_water - 8; - } else { - fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */ - fc->low_water = fc->high_water - 16; - } - - fc->pause_time = IGB_FC_PAUSE_TIME; - fc->send_xon = TRUE; - if (adapter->fc) - fc->requested_mode = adapter->fc; - else - fc->requested_mode = e1000_fc_default; - - /* Issue a global reset */ - e1000_reset_hw(hw); - E1000_WRITE_REG(hw, E1000_WUC, 0); - - /* Reset for AutoMediaDetect */ - if (adapter->flags & IGB_MEDIA_RESET) { - e1000_setup_init_funcs(hw, TRUE); - e1000_get_bus_info(hw); - adapter->flags &= ~IGB_MEDIA_RESET; - } - - if (e1000_init_hw(hw) < 0) - device_printf(dev, "Hardware Initialization Failed\n"); - - /* Setup DMA Coalescing */ - igb_init_dmac(adapter, pba); - - E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); - e1000_get_phy_info(hw); - e1000_check_for_link(hw); - return; -} - -/********************************************************************* - * - * Setup networking device structure and register an interface. - * - **********************************************************************/ -static int -igb_setup_interface(device_t dev, struct adapter *adapter) -{ - struct ifnet *ifp; - - INIT_DEBUGOUT("igb_setup_interface: begin"); - - ifp = adapter->ifp = if_alloc(IFT_ETHER); - if (ifp == NULL) { - device_printf(dev, "can not allocate ifnet structure\n"); - return (-1); - } - if_initname(ifp, device_get_name(dev), device_get_unit(dev)); - ifp->if_init = igb_init; - ifp->if_softc = adapter; - ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; - ifp->if_ioctl = igb_ioctl; - ifp->if_get_counter = igb_get_counter; - - /* TSO parameters */ - ifp->if_hw_tsomax = IP_MAXPACKET; - ifp->if_hw_tsomaxsegcount = IGB_MAX_SCATTER; - ifp->if_hw_tsomaxsegsize = IGB_TSO_SEG_SIZE; - -#ifndef IGB_LEGACY_TX - ifp->if_transmit = igb_mq_start; - ifp->if_qflush = igb_qflush; -#else - ifp->if_start = igb_start; - IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1); - ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1; - IFQ_SET_READY(&ifp->if_snd); -#endif - - ether_ifattach(ifp, adapter->hw.mac.addr); - - ifp->if_capabilities = ifp->if_capenable = 0; - - ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; -#if __FreeBSD_version >= 1000000 - ifp->if_capabilities |= IFCAP_HWCSUM_IPV6; -#endif - ifp->if_capabilities |= IFCAP_TSO; - ifp->if_capabilities |= IFCAP_JUMBO_MTU; - ifp->if_capenable = ifp->if_capabilities; - - /* Don't enable LRO by default */ - ifp->if_capabilities |= IFCAP_LRO; - -#ifdef DEVICE_POLLING - ifp->if_capabilities |= IFCAP_POLLING; -#endif - - /* - * Tell the upper layer(s) we - * support full VLAN capability. - */ - ifp->if_hdrlen = sizeof(struct ether_vlan_header); - ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING - | IFCAP_VLAN_HWTSO - | IFCAP_VLAN_MTU; - ifp->if_capenable |= IFCAP_VLAN_HWTAGGING - | IFCAP_VLAN_HWTSO - | IFCAP_VLAN_MTU; - - /* - ** Don't turn this on by default, if vlans are - ** created on another pseudo device (eg. lagg) - ** then vlan events are not passed thru, breaking - ** operation, but with HW FILTER off it works. If - ** using vlans directly on the igb driver you can - ** enable this and get full hardware tag filtering. - */ - ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; - - /* - * Specify the media types supported by this adapter and register - * callbacks to update media and link information - */ - ifmedia_init(&adapter->media, IFM_IMASK, - igb_media_change, igb_media_status); - if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || - (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { - ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX, - 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL); - } else { - ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, - 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, - 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, - 0, NULL); - if (adapter->hw.phy.type != e1000_phy_ife) { - ifmedia_add(&adapter->media, - IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); - ifmedia_add(&adapter->media, - IFM_ETHER | IFM_1000_T, 0, NULL); - } - } - ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); - ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); - return (0); -} - - -/* - * Manage DMA'able memory. - */ -static void -igb_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) -{ - if (error) - return; - *(bus_addr_t *) arg = segs[0].ds_addr; -} - -static int -igb_dma_malloc(struct adapter *adapter, bus_size_t size, - struct igb_dma_alloc *dma, int mapflags) -{ - int error; - - error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ - IGB_DBA_ALIGN, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - size, /* maxsize */ - 1, /* nsegments */ - size, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockarg */ - &dma->dma_tag); - if (error) { - device_printf(adapter->dev, - "%s: bus_dma_tag_create failed: %d\n", - __func__, error); - goto fail_0; - } - - error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, - BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map); - if (error) { - device_printf(adapter->dev, - "%s: bus_dmamem_alloc(%ju) failed: %d\n", - __func__, (uintmax_t)size, error); - goto fail_2; - } - - dma->dma_paddr = 0; - error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, - size, igb_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); - if (error || dma->dma_paddr == 0) { - device_printf(adapter->dev, - "%s: bus_dmamap_load failed: %d\n", - __func__, error); - goto fail_3; - } - - return (0); - -fail_3: - bus_dmamap_unload(dma->dma_tag, dma->dma_map); -fail_2: - bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); - bus_dma_tag_destroy(dma->dma_tag); -fail_0: - dma->dma_tag = NULL; - - return (error); -} - -static void -igb_dma_free(struct adapter *adapter, struct igb_dma_alloc *dma) -{ - if (dma->dma_tag == NULL) - return; - if (dma->dma_paddr != 0) { - bus_dmamap_sync(dma->dma_tag, dma->dma_map, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(dma->dma_tag, dma->dma_map); - dma->dma_paddr = 0; - } - if (dma->dma_vaddr != NULL) { - bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); - dma->dma_vaddr = NULL; - } - bus_dma_tag_destroy(dma->dma_tag); - dma->dma_tag = NULL; -} - - -/********************************************************************* - * - * Allocate memory for the transmit and receive rings, and then - * the descriptors associated with each, called only once at attach. - * - **********************************************************************/ -static int -igb_allocate_queues(struct adapter *adapter) -{ - device_t dev = adapter->dev; - struct igb_queue *que = NULL; - struct tx_ring *txr = NULL; - struct rx_ring *rxr = NULL; - int rsize, tsize, error = E1000_SUCCESS; - int txconf = 0, rxconf = 0; - - /* First allocate the top level queue structs */ - if (!(adapter->queues = - (struct igb_queue *) malloc(sizeof(struct igb_queue) * - adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate queue memory\n"); - error = ENOMEM; - goto fail; - } - - /* Next allocate the TX ring struct memory */ - if (!(adapter->tx_rings = - (struct tx_ring *) malloc(sizeof(struct tx_ring) * - adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate TX ring memory\n"); - error = ENOMEM; - goto tx_fail; - } - - /* Now allocate the RX */ - if (!(adapter->rx_rings = - (struct rx_ring *) malloc(sizeof(struct rx_ring) * - adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate RX ring memory\n"); - error = ENOMEM; - goto rx_fail; - } - - tsize = roundup2(adapter->num_tx_desc * - sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN); - /* - * Now set up the TX queues, txconf is needed to handle the - * possibility that things fail midcourse and we need to - * undo memory gracefully - */ - for (int i = 0; i < adapter->num_queues; i++, txconf++) { - /* Set up some basics */ - txr = &adapter->tx_rings[i]; - txr->adapter = adapter; - txr->me = i; - txr->num_desc = adapter->num_tx_desc; - - /* Initialize the TX lock */ - snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", - device_get_nameunit(dev), txr->me); - mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); - - if (igb_dma_malloc(adapter, tsize, - &txr->txdma, BUS_DMA_NOWAIT)) { - device_printf(dev, - "Unable to allocate TX Descriptor memory\n"); - error = ENOMEM; - goto err_tx_desc; - } - txr->tx_base = (union e1000_adv_tx_desc *)txr->txdma.dma_vaddr; - bzero((void *)txr->tx_base, tsize); - - /* Now allocate transmit buffers for the ring */ - if (igb_allocate_transmit_buffers(txr)) { - device_printf(dev, - "Critical Failure setting up transmit buffers\n"); - error = ENOMEM; - goto err_tx_desc; - } -#ifndef IGB_LEGACY_TX - /* Allocate a buf ring */ - txr->br = buf_ring_alloc(igb_buf_ring_size, M_DEVBUF, - M_WAITOK, &txr->tx_mtx); -#endif - } - - /* - * Next the RX queues... - */ - rsize = roundup2(adapter->num_rx_desc * - sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN); - for (int i = 0; i < adapter->num_queues; i++, rxconf++) { - rxr = &adapter->rx_rings[i]; - rxr->adapter = adapter; - rxr->me = i; - - /* Initialize the RX lock */ - snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", - device_get_nameunit(dev), txr->me); - mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); - - if (igb_dma_malloc(adapter, rsize, - &rxr->rxdma, BUS_DMA_NOWAIT)) { - device_printf(dev, - "Unable to allocate RxDescriptor memory\n"); - error = ENOMEM; - goto err_rx_desc; - } - rxr->rx_base = (union e1000_adv_rx_desc *)rxr->rxdma.dma_vaddr; - bzero((void *)rxr->rx_base, rsize); - - /* Allocate receive buffers for the ring*/ - if (igb_allocate_receive_buffers(rxr)) { - device_printf(dev, - "Critical Failure setting up receive buffers\n"); - error = ENOMEM; - goto err_rx_desc; - } - } - - /* - ** Finally set up the queue holding structs - */ - for (int i = 0; i < adapter->num_queues; i++) { - que = &adapter->queues[i]; - que->adapter = adapter; - que->txr = &adapter->tx_rings[i]; - que->rxr = &adapter->rx_rings[i]; - } - - return (0); - -err_rx_desc: - for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) - igb_dma_free(adapter, &rxr->rxdma); -err_tx_desc: - for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) - igb_dma_free(adapter, &txr->txdma); - free(adapter->rx_rings, M_DEVBUF); -rx_fail: -#ifndef IGB_LEGACY_TX - buf_ring_free(txr->br, M_DEVBUF); -#endif - free(adapter->tx_rings, M_DEVBUF); -tx_fail: - free(adapter->queues, M_DEVBUF); -fail: - return (error); -} - -/********************************************************************* - * - * Allocate memory for tx_buffer structures. The tx_buffer stores all - * the information needed to transmit a packet on the wire. This is - * called only once at attach, setup is done every reset. - * - **********************************************************************/ -static int -igb_allocate_transmit_buffers(struct tx_ring *txr) -{ - struct adapter *adapter = txr->adapter; - device_t dev = adapter->dev; - struct igb_tx_buf *txbuf; - int error, i; - - /* - * Setup DMA descriptor areas. - */ - if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - IGB_TSO_SIZE, /* maxsize */ - IGB_MAX_SCATTER, /* nsegments */ - PAGE_SIZE, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockfuncarg */ - &txr->txtag))) { - device_printf(dev,"Unable to allocate TX DMA tag\n"); - goto fail; - } - - if (!(txr->tx_buffers = - (struct igb_tx_buf *) malloc(sizeof(struct igb_tx_buf) * - adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate tx_buffer memory\n"); - error = ENOMEM; - goto fail; - } - - /* Create the descriptor buffer dma maps */ - txbuf = txr->tx_buffers; - for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { - error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); - if (error != 0) { - device_printf(dev, "Unable to create TX DMA map\n"); - goto fail; - } - } - - return 0; -fail: - /* We free all, it handles case where we are in the middle */ - igb_free_transmit_structures(adapter); - return (error); -} - -/********************************************************************* - * - * Initialize a transmit ring. - * - **********************************************************************/ -static void -igb_setup_transmit_ring(struct tx_ring *txr) -{ - struct adapter *adapter = txr->adapter; - struct igb_tx_buf *txbuf; - int i; -#ifdef DEV_NETMAP - struct netmap_adapter *na = NA(adapter->ifp); - struct netmap_slot *slot; -#endif /* DEV_NETMAP */ - - /* Clear the old descriptor contents */ - IGB_TX_LOCK(txr); -#ifdef DEV_NETMAP - slot = netmap_reset(na, NR_TX, txr->me, 0); -#endif /* DEV_NETMAP */ - bzero((void *)txr->tx_base, - (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc); - /* Reset indices */ - txr->next_avail_desc = 0; - txr->next_to_clean = 0; - - /* Free any existing tx buffers. */ - txbuf = txr->tx_buffers; - for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { - if (txbuf->m_head != NULL) { - bus_dmamap_sync(txr->txtag, txbuf->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(txr->txtag, txbuf->map); - m_freem(txbuf->m_head); - txbuf->m_head = NULL; - } -#ifdef DEV_NETMAP - if (slot) { - int si = netmap_idx_n2k(&na->tx_rings[txr->me], i); - /* no need to set the address */ - netmap_load_map(na, txr->txtag, txbuf->map, NMB(na, slot + si)); - } -#endif /* DEV_NETMAP */ - /* clear the watch index */ - txbuf->eop = NULL; - } - - /* Set number of descriptors available */ - txr->tx_avail = adapter->num_tx_desc; - - bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - IGB_TX_UNLOCK(txr); -} - -/********************************************************************* - * - * Initialize all transmit rings. - * - **********************************************************************/ -static void -igb_setup_transmit_structures(struct adapter *adapter) -{ - struct tx_ring *txr = adapter->tx_rings; - - for (int i = 0; i < adapter->num_queues; i++, txr++) - igb_setup_transmit_ring(txr); - - return; -} - -/********************************************************************* - * - * Enable transmit unit. - * - **********************************************************************/ -static void -igb_initialize_transmit_units(struct adapter *adapter) -{ - struct tx_ring *txr = adapter->tx_rings; - struct e1000_hw *hw = &adapter->hw; - u32 tctl, txdctl; - - INIT_DEBUGOUT("igb_initialize_transmit_units: begin"); - tctl = txdctl = 0; - - /* Setup the Tx Descriptor Rings */ - for (int i = 0; i < adapter->num_queues; i++, txr++) { - u64 bus_addr = txr->txdma.dma_paddr; - - E1000_WRITE_REG(hw, E1000_TDLEN(i), - adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); - E1000_WRITE_REG(hw, E1000_TDBAH(i), - (uint32_t)(bus_addr >> 32)); - E1000_WRITE_REG(hw, E1000_TDBAL(i), - (uint32_t)bus_addr); - - /* Setup the HW Tx Head and Tail descriptor pointers */ - E1000_WRITE_REG(hw, E1000_TDT(i), 0); - E1000_WRITE_REG(hw, E1000_TDH(i), 0); - - HW_DEBUGOUT2("Base = %x, Length = %x\n", - E1000_READ_REG(hw, E1000_TDBAL(i)), - E1000_READ_REG(hw, E1000_TDLEN(i))); - - txr->queue_status = IGB_QUEUE_IDLE; - - txdctl |= IGB_TX_PTHRESH; - txdctl |= IGB_TX_HTHRESH << 8; - txdctl |= IGB_TX_WTHRESH << 16; - txdctl |= E1000_TXDCTL_QUEUE_ENABLE; - E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl); - } - - if (adapter->vf_ifp) - return; - - e1000_config_collision_dist(hw); - - /* Program the Transmit Control Register */ - tctl = E1000_READ_REG(hw, E1000_TCTL); - tctl &= ~E1000_TCTL_CT; - tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | - (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); - - /* This write will effectively turn on the transmit unit. */ - E1000_WRITE_REG(hw, E1000_TCTL, tctl); -} - -/********************************************************************* - * - * Free all transmit rings. - * - **********************************************************************/ -static void -igb_free_transmit_structures(struct adapter *adapter) -{ - struct tx_ring *txr = adapter->tx_rings; - - for (int i = 0; i < adapter->num_queues; i++, txr++) { - IGB_TX_LOCK(txr); - igb_free_transmit_buffers(txr); - igb_dma_free(adapter, &txr->txdma); - IGB_TX_UNLOCK(txr); - IGB_TX_LOCK_DESTROY(txr); - } - free(adapter->tx_rings, M_DEVBUF); -} - -/********************************************************************* - * - * Free transmit ring related data structures. - * - **********************************************************************/ -static void -igb_free_transmit_buffers(struct tx_ring *txr) -{ - struct adapter *adapter = txr->adapter; - struct igb_tx_buf *tx_buffer; - int i; - - INIT_DEBUGOUT("free_transmit_ring: begin"); - - if (txr->tx_buffers == NULL) - return; - - tx_buffer = txr->tx_buffers; - for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { - if (tx_buffer->m_head != NULL) { - bus_dmamap_sync(txr->txtag, tx_buffer->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(txr->txtag, - tx_buffer->map); - m_freem(tx_buffer->m_head); - tx_buffer->m_head = NULL; - if (tx_buffer->map != NULL) { - bus_dmamap_destroy(txr->txtag, - tx_buffer->map); - tx_buffer->map = NULL; - } - } else if (tx_buffer->map != NULL) { - bus_dmamap_unload(txr->txtag, - tx_buffer->map); - bus_dmamap_destroy(txr->txtag, - tx_buffer->map); - tx_buffer->map = NULL; - } - } -#ifndef IGB_LEGACY_TX - if (txr->br != NULL) - buf_ring_free(txr->br, M_DEVBUF); -#endif - if (txr->tx_buffers != NULL) { - free(txr->tx_buffers, M_DEVBUF); - txr->tx_buffers = NULL; - } - if (txr->txtag != NULL) { - bus_dma_tag_destroy(txr->txtag); - txr->txtag = NULL; - } - return; -} - -/********************************************************************** - * - * Setup work for hardware segmentation offload (TSO) on - * adapters using advanced tx descriptors - * - **********************************************************************/ -static int -igb_tso_setup(struct tx_ring *txr, struct mbuf *mp, - u32 *cmd_type_len, u32 *olinfo_status) -{ - struct adapter *adapter = txr->adapter; - struct e1000_adv_tx_context_desc *TXD; - u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; - u32 mss_l4len_idx = 0, paylen; - u16 vtag = 0, eh_type; - int ctxd, ehdrlen, ip_hlen, tcp_hlen; - struct ether_vlan_header *eh; -#ifdef INET6 - struct ip6_hdr *ip6; -#endif -#ifdef INET - struct ip *ip; -#endif - struct tcphdr *th; - - - /* - * Determine where frame payload starts. - * Jump over vlan headers if already present - */ - eh = mtod(mp, struct ether_vlan_header *); - if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { - ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; - eh_type = eh->evl_proto; - } else { - ehdrlen = ETHER_HDR_LEN; - eh_type = eh->evl_encap_proto; - } - - switch (ntohs(eh_type)) { -#ifdef INET6 - case ETHERTYPE_IPV6: - ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); - /* XXX-BZ For now we do not pretend to support ext. hdrs. */ - if (ip6->ip6_nxt != IPPROTO_TCP) - return (ENXIO); - ip_hlen = sizeof(struct ip6_hdr); - ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); - th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); - th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); - type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6; - break; -#endif -#ifdef INET - case ETHERTYPE_IP: - ip = (struct ip *)(mp->m_data + ehdrlen); - if (ip->ip_p != IPPROTO_TCP) - return (ENXIO); - ip->ip_sum = 0; - ip_hlen = ip->ip_hl << 2; - th = (struct tcphdr *)((caddr_t)ip + ip_hlen); - th->th_sum = in_pseudo(ip->ip_src.s_addr, - ip->ip_dst.s_addr, htons(IPPROTO_TCP)); - type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; - /* Tell transmit desc to also do IPv4 checksum. */ - *olinfo_status |= E1000_TXD_POPTS_IXSM << 8; - break; -#endif - default: - panic("%s: CSUM_TSO but no supported IP version (0x%04x)", - __func__, ntohs(eh_type)); - break; - } - - ctxd = txr->next_avail_desc; - TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd]; - - tcp_hlen = th->th_off << 2; - - /* This is used in the transmit desc in encap */ - paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen; - - /* VLAN MACLEN IPLEN */ - if (mp->m_flags & M_VLANTAG) { - vtag = htole16(mp->m_pkthdr.ether_vtag); - vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT); - } - - vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT; - vlan_macip_lens |= ip_hlen; - TXD->vlan_macip_lens = htole32(vlan_macip_lens); - - /* ADV DTYPE TUCMD */ - type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; - type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; - TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); - - /* MSS L4LEN IDX */ - mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << E1000_ADVTXD_MSS_SHIFT); - mss_l4len_idx |= (tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT); - /* 82575 needs the queue index added */ - if (adapter->hw.mac.type == e1000_82575) - mss_l4len_idx |= txr->me << 4; - TXD->mss_l4len_idx = htole32(mss_l4len_idx); - - TXD->seqnum_seed = htole32(0); - - if (++ctxd == txr->num_desc) - ctxd = 0; - - txr->tx_avail--; - txr->next_avail_desc = ctxd; - *cmd_type_len |= E1000_ADVTXD_DCMD_TSE; - *olinfo_status |= E1000_TXD_POPTS_TXSM << 8; - *olinfo_status |= paylen << E1000_ADVTXD_PAYLEN_SHIFT; - ++txr->tso_tx; - return (0); -} - -/********************************************************************* - * - * Advanced Context Descriptor setup for VLAN, CSUM or TSO - * - **********************************************************************/ - -static int -igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, - u32 *cmd_type_len, u32 *olinfo_status) -{ - struct e1000_adv_tx_context_desc *TXD; - struct adapter *adapter = txr->adapter; - struct ether_vlan_header *eh; - struct ip *ip; - struct ip6_hdr *ip6; - u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0, mss_l4len_idx = 0; - int ehdrlen, ip_hlen = 0; - u16 etype; - u8 ipproto = 0; - int offload = TRUE; - int ctxd = txr->next_avail_desc; - u16 vtag = 0; - - /* First check if TSO is to be used */ - if (mp->m_pkthdr.csum_flags & CSUM_TSO) - return (igb_tso_setup(txr, mp, cmd_type_len, olinfo_status)); - - if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) - offload = FALSE; - - /* Indicate the whole packet as payload when not doing TSO */ - *olinfo_status |= mp->m_pkthdr.len << E1000_ADVTXD_PAYLEN_SHIFT; - - /* Now ready a context descriptor */ - TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd]; - - /* - ** In advanced descriptors the vlan tag must - ** be placed into the context descriptor. Hence - ** we need to make one even if not doing offloads. - */ - if (mp->m_flags & M_VLANTAG) { - vtag = htole16(mp->m_pkthdr.ether_vtag); - vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT); - } else if (offload == FALSE) /* ... no offload to do */ - return (0); - - /* - * Determine where frame payload starts. - * Jump over vlan headers if already present, - * helpful for QinQ too. - */ - eh = mtod(mp, struct ether_vlan_header *); - if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { - etype = ntohs(eh->evl_proto); - ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; - } else { - etype = ntohs(eh->evl_encap_proto); - ehdrlen = ETHER_HDR_LEN; - } - - /* Set the ether header length */ - vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT; - - switch (etype) { - case ETHERTYPE_IP: - ip = (struct ip *)(mp->m_data + ehdrlen); - ip_hlen = ip->ip_hl << 2; - ipproto = ip->ip_p; - type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; - break; - case ETHERTYPE_IPV6: - ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); - ip_hlen = sizeof(struct ip6_hdr); - /* XXX-BZ this will go badly in case of ext hdrs. */ - ipproto = ip6->ip6_nxt; - type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6; - break; - default: - offload = FALSE; - break; - } - - vlan_macip_lens |= ip_hlen; - type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; - - switch (ipproto) { - case IPPROTO_TCP: -#if __FreeBSD_version >= 1000000 - if (mp->m_pkthdr.csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP)) -#else - if (mp->m_pkthdr.csum_flags & CSUM_TCP) -#endif - type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; - break; - case IPPROTO_UDP: -#if __FreeBSD_version >= 1000000 - if (mp->m_pkthdr.csum_flags & (CSUM_IP_UDP | CSUM_IP6_UDP)) -#else - if (mp->m_pkthdr.csum_flags & CSUM_UDP) -#endif - type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP; - break; - -#if __FreeBSD_version >= 800000 - case IPPROTO_SCTP: -#if __FreeBSD_version >= 1000000 - if (mp->m_pkthdr.csum_flags & (CSUM_IP_SCTP | CSUM_IP6_SCTP)) -#else - if (mp->m_pkthdr.csum_flags & CSUM_SCTP) -#endif - type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP; - break; -#endif - default: - offload = FALSE; - break; - } - - if (offload) /* For the TX descriptor setup */ - *olinfo_status |= E1000_TXD_POPTS_TXSM << 8; - - /* 82575 needs the queue index added */ - if (adapter->hw.mac.type == e1000_82575) - mss_l4len_idx = txr->me << 4; - - /* Now copy bits into descriptor */ - TXD->vlan_macip_lens = htole32(vlan_macip_lens); - TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); - TXD->seqnum_seed = htole32(0); - TXD->mss_l4len_idx = htole32(mss_l4len_idx); - - /* We've consumed the first desc, adjust counters */ - if (++ctxd == txr->num_desc) - ctxd = 0; - txr->next_avail_desc = ctxd; - --txr->tx_avail; - - return (0); -} - -/********************************************************************** - * - * Examine each tx_buffer in the used queue. If the hardware is done - * processing the packet then free associated resources. The - * tx_buffer is put back on the free queue. - * - * TRUE return means there's work in the ring to clean, FALSE its empty. - **********************************************************************/ -static bool -igb_txeof(struct tx_ring *txr) -{ - struct adapter *adapter = txr->adapter; -#ifdef DEV_NETMAP - struct ifnet *ifp = adapter->ifp; -#endif /* DEV_NETMAP */ - u32 work, processed = 0; - int limit = adapter->tx_process_limit; - struct igb_tx_buf *buf; - union e1000_adv_tx_desc *txd; - - mtx_assert(&txr->tx_mtx, MA_OWNED); - -#ifdef DEV_NETMAP - if (netmap_tx_irq(ifp, txr->me)) - return (FALSE); -#endif /* DEV_NETMAP */ - - if (txr->tx_avail == txr->num_desc) { - txr->queue_status = IGB_QUEUE_IDLE; - return FALSE; - } - - /* Get work starting point */ - work = txr->next_to_clean; - buf = &txr->tx_buffers[work]; - txd = &txr->tx_base[work]; - work -= txr->num_desc; /* The distance to ring end */ - bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - do { - union e1000_adv_tx_desc *eop = buf->eop; - if (eop == NULL) /* No work */ - break; - - if ((eop->wb.status & E1000_TXD_STAT_DD) == 0) - break; /* I/O not complete */ - - if (buf->m_head) { - txr->bytes += - buf->m_head->m_pkthdr.len; - bus_dmamap_sync(txr->txtag, - buf->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(txr->txtag, - buf->map); - m_freem(buf->m_head); - buf->m_head = NULL; - } - buf->eop = NULL; - ++txr->tx_avail; - - /* We clean the range if multi segment */ - while (txd != eop) { - ++txd; - ++buf; - ++work; - /* wrap the ring? */ - if (__predict_false(!work)) { - work -= txr->num_desc; - buf = txr->tx_buffers; - txd = txr->tx_base; - } - if (buf->m_head) { - txr->bytes += - buf->m_head->m_pkthdr.len; - bus_dmamap_sync(txr->txtag, - buf->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(txr->txtag, - buf->map); - m_freem(buf->m_head); - buf->m_head = NULL; - } - ++txr->tx_avail; - buf->eop = NULL; - - } - ++txr->packets; - ++processed; - txr->watchdog_time = ticks; - - /* Try the next packet */ - ++txd; - ++buf; - ++work; - /* reset with a wrap */ - if (__predict_false(!work)) { - work -= txr->num_desc; - buf = txr->tx_buffers; - txd = txr->tx_base; - } - prefetch(txd); - } while (__predict_true(--limit)); - - bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - - work += txr->num_desc; - txr->next_to_clean = work; - - /* - ** Watchdog calculation, we know there's - ** work outstanding or the first return - ** would have been taken, so none processed - ** for too long indicates a hang. - */ - if ((!processed) && ((ticks - txr->watchdog_time) > IGB_WATCHDOG)) - txr->queue_status |= IGB_QUEUE_HUNG; - - if (txr->tx_avail >= IGB_QUEUE_THRESHOLD) - txr->queue_status &= ~IGB_QUEUE_DEPLETED; - - if (txr->tx_avail == txr->num_desc) { - txr->queue_status = IGB_QUEUE_IDLE; - return (FALSE); - } - - return (TRUE); -} - -/********************************************************************* - * - * Refresh mbuf buffers for RX descriptor rings - * - now keeps its own state so discards due to resource - * exhaustion are unnecessary, if an mbuf cannot be obtained - * it just returns, keeping its placeholder, thus it can simply - * be recalled to try again. - * - **********************************************************************/ -static void -igb_refresh_mbufs(struct rx_ring *rxr, int limit) -{ - struct adapter *adapter = rxr->adapter; - bus_dma_segment_t hseg[1]; - bus_dma_segment_t pseg[1]; - struct igb_rx_buf *rxbuf; - struct mbuf *mh, *mp; - int i, j, nsegs, error; - bool refreshed = FALSE; - - i = j = rxr->next_to_refresh; - /* - ** Get one descriptor beyond - ** our work mark to control - ** the loop. - */ - if (++j == adapter->num_rx_desc) - j = 0; - - while (j != limit) { - rxbuf = &rxr->rx_buffers[i]; - /* No hdr mbuf used with header split off */ - if (rxr->hdr_split == FALSE) - goto no_split; - if (rxbuf->m_head == NULL) { - mh = m_gethdr(M_NOWAIT, MT_DATA); - if (mh == NULL) - goto update; - } else - mh = rxbuf->m_head; - - mh->m_pkthdr.len = mh->m_len = MHLEN; - mh->m_len = MHLEN; - mh->m_flags |= M_PKTHDR; - /* Get the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->htag, - rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT); - if (error != 0) { - printf("Refresh mbufs: hdr dmamap load" - " failure - %d\n", error); - m_free(mh); - rxbuf->m_head = NULL; - goto update; - } - rxbuf->m_head = mh; - bus_dmamap_sync(rxr->htag, rxbuf->hmap, - BUS_DMASYNC_PREREAD); - rxr->rx_base[i].read.hdr_addr = - htole64(hseg[0].ds_addr); -no_split: - if (rxbuf->m_pack == NULL) { - mp = m_getjcl(M_NOWAIT, MT_DATA, - M_PKTHDR, adapter->rx_mbuf_sz); - if (mp == NULL) - goto update; - } else - mp = rxbuf->m_pack; - - mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; - /* Get the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->ptag, - rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT); - if (error != 0) { - printf("Refresh mbufs: payload dmamap load" - " failure - %d\n", error); - m_free(mp); - rxbuf->m_pack = NULL; - goto update; - } - rxbuf->m_pack = mp; - bus_dmamap_sync(rxr->ptag, rxbuf->pmap, - BUS_DMASYNC_PREREAD); - rxr->rx_base[i].read.pkt_addr = - htole64(pseg[0].ds_addr); - refreshed = TRUE; /* I feel wefreshed :) */ - - i = j; /* our next is precalculated */ - rxr->next_to_refresh = i; - if (++j == adapter->num_rx_desc) - j = 0; - } -update: - if (refreshed) /* update tail */ - E1000_WRITE_REG(&adapter->hw, - E1000_RDT(rxr->me), rxr->next_to_refresh); - return; -} - - -/********************************************************************* - * - * Allocate memory for rx_buffer structures. Since we use one - * rx_buffer per received packet, the maximum number of rx_buffer's - * that we'll need is equal to the number of receive descriptors - * that we've allocated. - * - **********************************************************************/ -static int -igb_allocate_receive_buffers(struct rx_ring *rxr) -{ - struct adapter *adapter = rxr->adapter; - device_t dev = adapter->dev; - struct igb_rx_buf *rxbuf; - int i, bsize, error; - - bsize = sizeof(struct igb_rx_buf) * adapter->num_rx_desc; - if (!(rxr->rx_buffers = - (struct igb_rx_buf *) malloc(bsize, - M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate rx_buffer memory\n"); - error = ENOMEM; - goto fail; - } - - if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - MSIZE, /* maxsize */ - 1, /* nsegments */ - MSIZE, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockfuncarg */ - &rxr->htag))) { - device_printf(dev, "Unable to create RX DMA tag\n"); - goto fail; - } - - if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - MJUM9BYTES, /* maxsize */ - 1, /* nsegments */ - MJUM9BYTES, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockfuncarg */ - &rxr->ptag))) { - device_printf(dev, "Unable to create RX payload DMA tag\n"); - goto fail; - } - - for (i = 0; i < adapter->num_rx_desc; i++) { - rxbuf = &rxr->rx_buffers[i]; - error = bus_dmamap_create(rxr->htag, 0, &rxbuf->hmap); - if (error) { - device_printf(dev, - "Unable to create RX head DMA maps\n"); - goto fail; - } - error = bus_dmamap_create(rxr->ptag, 0, &rxbuf->pmap); - if (error) { - device_printf(dev, - "Unable to create RX packet DMA maps\n"); - goto fail; - } - } - - return (0); - -fail: - /* Frees all, but can handle partial completion */ - igb_free_receive_structures(adapter); - return (error); -} - - -static void -igb_free_receive_ring(struct rx_ring *rxr) -{ - struct adapter *adapter = rxr->adapter; - struct igb_rx_buf *rxbuf; - - - for (int i = 0; i < adapter->num_rx_desc; i++) { - rxbuf = &rxr->rx_buffers[i]; - if (rxbuf->m_head != NULL) { - bus_dmamap_sync(rxr->htag, rxbuf->hmap, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->htag, rxbuf->hmap); - rxbuf->m_head->m_flags |= M_PKTHDR; - m_freem(rxbuf->m_head); - } - if (rxbuf->m_pack != NULL) { - bus_dmamap_sync(rxr->ptag, rxbuf->pmap, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->ptag, rxbuf->pmap); - rxbuf->m_pack->m_flags |= M_PKTHDR; - m_freem(rxbuf->m_pack); - } - rxbuf->m_head = NULL; - rxbuf->m_pack = NULL; - } -} - - -/********************************************************************* - * - * Initialize a receive ring and its buffers. - * - **********************************************************************/ -static int -igb_setup_receive_ring(struct rx_ring *rxr) -{ - struct adapter *adapter; - struct ifnet *ifp; - device_t dev; - struct igb_rx_buf *rxbuf; - bus_dma_segment_t pseg[1], hseg[1]; - struct lro_ctrl *lro = &rxr->lro; - int rsize, nsegs, error = 0; -#ifdef DEV_NETMAP - struct netmap_adapter *na = NA(rxr->adapter->ifp); - struct netmap_slot *slot; -#endif /* DEV_NETMAP */ - - adapter = rxr->adapter; - dev = adapter->dev; - ifp = adapter->ifp; - - /* Clear the ring contents */ - IGB_RX_LOCK(rxr); -#ifdef DEV_NETMAP - slot = netmap_reset(na, NR_RX, rxr->me, 0); -#endif /* DEV_NETMAP */ - rsize = roundup2(adapter->num_rx_desc * - sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN); - bzero((void *)rxr->rx_base, rsize); - - /* - ** Free current RX buffer structures and their mbufs - */ - igb_free_receive_ring(rxr); - - /* Configure for header split? */ - if (igb_header_split) - rxr->hdr_split = TRUE; - - /* Now replenish the ring mbufs */ - for (int j = 0; j < adapter->num_rx_desc; ++j) { - struct mbuf *mh, *mp; - - rxbuf = &rxr->rx_buffers[j]; -#ifdef DEV_NETMAP - if (slot) { - /* slot sj is mapped to the j-th NIC-ring entry */ - int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j); - uint64_t paddr; - void *addr; - - addr = PNMB(na, slot + sj, &paddr); - netmap_load_map(na, rxr->ptag, rxbuf->pmap, addr); - /* Update descriptor */ - rxr->rx_base[j].read.pkt_addr = htole64(paddr); - continue; - } -#endif /* DEV_NETMAP */ - if (rxr->hdr_split == FALSE) - goto skip_head; - - /* First the header */ - rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA); - if (rxbuf->m_head == NULL) { - error = ENOBUFS; - goto fail; - } - m_adj(rxbuf->m_head, ETHER_ALIGN); - mh = rxbuf->m_head; - mh->m_len = mh->m_pkthdr.len = MHLEN; - mh->m_flags |= M_PKTHDR; - /* Get the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->htag, - rxbuf->hmap, rxbuf->m_head, hseg, - &nsegs, BUS_DMA_NOWAIT); - if (error != 0) /* Nothing elegant to do here */ - goto fail; - bus_dmamap_sync(rxr->htag, - rxbuf->hmap, BUS_DMASYNC_PREREAD); - /* Update descriptor */ - rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr); - -skip_head: - /* Now the payload cluster */ - rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA, - M_PKTHDR, adapter->rx_mbuf_sz); - if (rxbuf->m_pack == NULL) { - error = ENOBUFS; - goto fail; - } - mp = rxbuf->m_pack; - mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; - /* Get the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->ptag, - rxbuf->pmap, mp, pseg, - &nsegs, BUS_DMA_NOWAIT); - if (error != 0) - goto fail; - bus_dmamap_sync(rxr->ptag, - rxbuf->pmap, BUS_DMASYNC_PREREAD); - /* Update descriptor */ - rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr); - } - - /* Setup our descriptor indices */ - rxr->next_to_check = 0; - rxr->next_to_refresh = adapter->num_rx_desc - 1; - rxr->lro_enabled = FALSE; - rxr->rx_split_packets = 0; - rxr->rx_bytes = 0; - - rxr->fmp = NULL; - rxr->lmp = NULL; - - bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - - /* - ** Now set up the LRO interface, we - ** also only do head split when LRO - ** is enabled, since so often they - ** are undesirable in similar setups. - */ - if (ifp->if_capenable & IFCAP_LRO) { - error = tcp_lro_init(lro); - if (error) { - device_printf(dev, "LRO Initialization failed!\n"); - goto fail; - } - INIT_DEBUGOUT("RX LRO Initialized\n"); - rxr->lro_enabled = TRUE; - lro->ifp = adapter->ifp; - } - - IGB_RX_UNLOCK(rxr); - return (0); - -fail: - igb_free_receive_ring(rxr); - IGB_RX_UNLOCK(rxr); - return (error); -} - - -/********************************************************************* - * - * Initialize all receive rings. - * - **********************************************************************/ -static int -igb_setup_receive_structures(struct adapter *adapter) -{ - struct rx_ring *rxr = adapter->rx_rings; - int i; - - for (i = 0; i < adapter->num_queues; i++, rxr++) - if (igb_setup_receive_ring(rxr)) - goto fail; - - return (0); -fail: - /* - * Free RX buffers allocated so far, we will only handle - * the rings that completed, the failing case will have - * cleaned up for itself. 'i' is the endpoint. - */ - for (int j = 0; j < i; ++j) { - rxr = &adapter->rx_rings[j]; - IGB_RX_LOCK(rxr); - igb_free_receive_ring(rxr); - IGB_RX_UNLOCK(rxr); - } - - return (ENOBUFS); -} - -/* - * Initialise the RSS mapping for NICs that support multiple transmit/ - * receive rings. - */ -static void -igb_initialise_rss_mapping(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - int i; - int queue_id; - u32 reta; - u32 rss_key[10], mrqc, shift = 0; - - /* XXX? */ - if (adapter->hw.mac.type == e1000_82575) - shift = 6; - - /* - * The redirection table controls which destination - * queue each bucket redirects traffic to. - * Each DWORD represents four queues, with the LSB - * being the first queue in the DWORD. - * - * This just allocates buckets to queues using round-robin - * allocation. - * - * NOTE: It Just Happens to line up with the default - * RSS allocation method. - */ - - /* Warning FM follows */ - reta = 0; - for (i = 0; i < 128; i++) { -#ifdef RSS - queue_id = rss_get_indirection_to_bucket(i); - /* - * If we have more queues than buckets, we'll - * end up mapping buckets to a subset of the - * queues. - * - * If we have more buckets than queues, we'll - * end up instead assigning multiple buckets - * to queues. - * - * Both are suboptimal, but we need to handle - * the case so we don't go out of bounds - * indexing arrays and such. - */ - queue_id = queue_id % adapter->num_queues; -#else - queue_id = (i % adapter->num_queues); -#endif - /* Adjust if required */ - queue_id = queue_id << shift; - - /* - * The low 8 bits are for hash value (n+0); - * The next 8 bits are for hash value (n+1), etc. - */ - reta = reta >> 8; - reta = reta | ( ((uint32_t) queue_id) << 24); - if ((i & 3) == 3) { - E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta); - reta = 0; - } - } - - /* Now fill in hash table */ - - /* - * MRQC: Multiple Receive Queues Command - * Set queuing to RSS control, number depends on the device. - */ - mrqc = E1000_MRQC_ENABLE_RSS_8Q; - -#ifdef RSS - /* XXX ew typecasting */ - rss_getkey((uint8_t *) &rss_key); -#else - arc4rand(&rss_key, sizeof(rss_key), 0); -#endif - for (i = 0; i < 10; i++) - E1000_WRITE_REG_ARRAY(hw, - E1000_RSSRK(0), i, rss_key[i]); - - /* - * Configure the RSS fields to hash upon. - */ - mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | - E1000_MRQC_RSS_FIELD_IPV4_TCP); - mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 | - E1000_MRQC_RSS_FIELD_IPV6_TCP); - mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP | - E1000_MRQC_RSS_FIELD_IPV6_UDP); - mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | - E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); - - E1000_WRITE_REG(hw, E1000_MRQC, mrqc); -} - -/********************************************************************* - * - * Enable receive unit. - * - **********************************************************************/ -static void -igb_initialize_receive_units(struct adapter *adapter) -{ - struct rx_ring *rxr = adapter->rx_rings; - struct ifnet *ifp = adapter->ifp; - struct e1000_hw *hw = &adapter->hw; - u32 rctl, rxcsum, psize, srrctl = 0; - - INIT_DEBUGOUT("igb_initialize_receive_unit: begin"); - - /* - * Make sure receives are disabled while setting - * up the descriptor ring - */ - rctl = E1000_READ_REG(hw, E1000_RCTL); - E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); - - /* - ** Set up for header split - */ - if (igb_header_split) { - /* Use a standard mbuf for the header */ - srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT; - srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; - } else - srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF; - - /* - ** Set up for jumbo frames - */ - if (ifp->if_mtu > ETHERMTU) { - rctl |= E1000_RCTL_LPE; - if (adapter->rx_mbuf_sz == MJUMPAGESIZE) { - srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT; - rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX; - } else if (adapter->rx_mbuf_sz > MJUMPAGESIZE) { - srrctl |= 8192 >> E1000_SRRCTL_BSIZEPKT_SHIFT; - rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX; - } - /* Set maximum packet len */ - psize = adapter->max_frame_size; - /* are we on a vlan? */ - if (adapter->ifp->if_vlantrunk != NULL) - psize += VLAN_TAG_SIZE; - E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize); - } else { - rctl &= ~E1000_RCTL_LPE; - srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT; - rctl |= E1000_RCTL_SZ_2048; - } - - /* - * If TX flow control is disabled and there's >1 queue defined, - * enable DROP. - * - * This drops frames rather than hanging the RX MAC for all queues. - */ - if ((adapter->num_queues > 1) && - (adapter->fc == e1000_fc_none || - adapter->fc == e1000_fc_rx_pause)) { - srrctl |= E1000_SRRCTL_DROP_EN; - } - - /* Setup the Base and Length of the Rx Descriptor Rings */ - for (int i = 0; i < adapter->num_queues; i++, rxr++) { - u64 bus_addr = rxr->rxdma.dma_paddr; - u32 rxdctl; - - E1000_WRITE_REG(hw, E1000_RDLEN(i), - adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); - E1000_WRITE_REG(hw, E1000_RDBAH(i), - (uint32_t)(bus_addr >> 32)); - E1000_WRITE_REG(hw, E1000_RDBAL(i), - (uint32_t)bus_addr); - E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl); - /* Enable this Queue */ - rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); - rxdctl |= E1000_RXDCTL_QUEUE_ENABLE; - rxdctl &= 0xFFF00000; - rxdctl |= IGB_RX_PTHRESH; - rxdctl |= IGB_RX_HTHRESH << 8; - rxdctl |= IGB_RX_WTHRESH << 16; - E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); - } - - /* - ** Setup for RX MultiQueue - */ - rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); - if (adapter->num_queues >1) { - - /* rss setup */ - igb_initialise_rss_mapping(adapter); - - /* - ** NOTE: Receive Full-Packet Checksum Offload - ** is mutually exclusive with Multiqueue. However - ** this is not the same as TCP/IP checksums which - ** still work. - */ - rxcsum |= E1000_RXCSUM_PCSD; -#if __FreeBSD_version >= 800000 - /* For SCTP Offload */ - if ((hw->mac.type != e1000_82575) && - (ifp->if_capenable & IFCAP_RXCSUM)) - rxcsum |= E1000_RXCSUM_CRCOFL; -#endif - } else { - /* Non RSS setup */ - if (ifp->if_capenable & IFCAP_RXCSUM) { - rxcsum |= E1000_RXCSUM_IPPCSE; -#if __FreeBSD_version >= 800000 - if (adapter->hw.mac.type != e1000_82575) - rxcsum |= E1000_RXCSUM_CRCOFL; -#endif - } else - rxcsum &= ~E1000_RXCSUM_TUOFL; - } - E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); - - /* Setup the Receive Control Register */ - rctl &= ~(3 << E1000_RCTL_MO_SHIFT); - rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO | - E1000_RCTL_RDMTS_HALF | - (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT); - /* Strip CRC bytes. */ - rctl |= E1000_RCTL_SECRC; - /* Make sure VLAN Filters are off */ - rctl &= ~E1000_RCTL_VFE; - /* Don't store bad packets */ - rctl &= ~E1000_RCTL_SBP; - - /* Enable Receives */ - E1000_WRITE_REG(hw, E1000_RCTL, rctl); - - /* - * Setup the HW Rx Head and Tail Descriptor Pointers - * - needs to be after enable - */ - for (int i = 0; i < adapter->num_queues; i++) { - rxr = &adapter->rx_rings[i]; - E1000_WRITE_REG(hw, E1000_RDH(i), rxr->next_to_check); -#ifdef DEV_NETMAP - /* - * an init() while a netmap client is active must - * preserve the rx buffers passed to userspace. - * In this driver it means we adjust RDT to - * something different from next_to_refresh - * (which is not used in netmap mode). - */ - if (ifp->if_capenable & IFCAP_NETMAP) { - struct netmap_adapter *na = NA(adapter->ifp); - struct netmap_kring *kring = &na->rx_rings[i]; - int t = rxr->next_to_refresh - nm_kr_rxspace(kring); - - if (t >= adapter->num_rx_desc) - t -= adapter->num_rx_desc; - else if (t < 0) - t += adapter->num_rx_desc; - E1000_WRITE_REG(hw, E1000_RDT(i), t); - } else -#endif /* DEV_NETMAP */ - E1000_WRITE_REG(hw, E1000_RDT(i), rxr->next_to_refresh); - } - return; -} - -/********************************************************************* - * - * Free receive rings. - * - **********************************************************************/ -static void -igb_free_receive_structures(struct adapter *adapter) -{ - struct rx_ring *rxr = adapter->rx_rings; - - for (int i = 0; i < adapter->num_queues; i++, rxr++) { - struct lro_ctrl *lro = &rxr->lro; - igb_free_receive_buffers(rxr); - tcp_lro_free(lro); - igb_dma_free(adapter, &rxr->rxdma); - } - - free(adapter->rx_rings, M_DEVBUF); -} - -/********************************************************************* - * - * Free receive ring data structures. - * - **********************************************************************/ -static void -igb_free_receive_buffers(struct rx_ring *rxr) -{ - struct adapter *adapter = rxr->adapter; - struct igb_rx_buf *rxbuf; - int i; - - INIT_DEBUGOUT("free_receive_structures: begin"); - - /* Cleanup any existing buffers */ - if (rxr->rx_buffers != NULL) { - for (i = 0; i < adapter->num_rx_desc; i++) { - rxbuf = &rxr->rx_buffers[i]; - if (rxbuf->m_head != NULL) { - bus_dmamap_sync(rxr->htag, rxbuf->hmap, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->htag, rxbuf->hmap); - rxbuf->m_head->m_flags |= M_PKTHDR; - m_freem(rxbuf->m_head); - } - if (rxbuf->m_pack != NULL) { - bus_dmamap_sync(rxr->ptag, rxbuf->pmap, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->ptag, rxbuf->pmap); - rxbuf->m_pack->m_flags |= M_PKTHDR; - m_freem(rxbuf->m_pack); - } - rxbuf->m_head = NULL; - rxbuf->m_pack = NULL; - if (rxbuf->hmap != NULL) { - bus_dmamap_destroy(rxr->htag, rxbuf->hmap); - rxbuf->hmap = NULL; - } - if (rxbuf->pmap != NULL) { - bus_dmamap_destroy(rxr->ptag, rxbuf->pmap); - rxbuf->pmap = NULL; - } - } - if (rxr->rx_buffers != NULL) { - free(rxr->rx_buffers, M_DEVBUF); - rxr->rx_buffers = NULL; - } - } - - if (rxr->htag != NULL) { - bus_dma_tag_destroy(rxr->htag); - rxr->htag = NULL; - } - if (rxr->ptag != NULL) { - bus_dma_tag_destroy(rxr->ptag); - rxr->ptag = NULL; - } -} - -static __inline void -igb_rx_discard(struct rx_ring *rxr, int i) -{ - struct igb_rx_buf *rbuf; - - rbuf = &rxr->rx_buffers[i]; - - /* Partially received? Free the chain */ - if (rxr->fmp != NULL) { - rxr->fmp->m_flags |= M_PKTHDR; - m_freem(rxr->fmp); - rxr->fmp = NULL; - rxr->lmp = NULL; - } - - /* - ** With advanced descriptors the writeback - ** clobbers the buffer addrs, so its easier - ** to just free the existing mbufs and take - ** the normal refresh path to get new buffers - ** and mapping. - */ - if (rbuf->m_head) { - m_free(rbuf->m_head); - rbuf->m_head = NULL; - bus_dmamap_unload(rxr->htag, rbuf->hmap); - } - - if (rbuf->m_pack) { - m_free(rbuf->m_pack); - rbuf->m_pack = NULL; - bus_dmamap_unload(rxr->ptag, rbuf->pmap); - } - - return; -} - -static __inline void -igb_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) -{ - - /* - * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet - * should be computed by hardware. Also it should not have VLAN tag in - * ethernet header. - */ - if (rxr->lro_enabled && - (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && - (ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 && - (ptype & (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP)) == - (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP) && - (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == - (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { - /* - * Send to the stack if: - ** - LRO not enabled, or - ** - no LRO resources, or - ** - lro enqueue fails - */ - if (rxr->lro.lro_cnt != 0) - if (tcp_lro_rx(&rxr->lro, m, 0) == 0) - return; - } - IGB_RX_UNLOCK(rxr); - (*ifp->if_input)(ifp, m); - IGB_RX_LOCK(rxr); -} - -/********************************************************************* - * - * This routine executes in interrupt context. It replenishes - * the mbufs in the descriptor and sends data which has been - * dma'ed into host memory to upper layer. - * - * We loop at most count times if count is > 0, or until done if - * count < 0. - * - * Return TRUE if more to clean, FALSE otherwise - *********************************************************************/ -static bool -igb_rxeof(struct igb_queue *que, int count, int *done) -{ - struct adapter *adapter = que->adapter; - struct rx_ring *rxr = que->rxr; - struct ifnet *ifp = adapter->ifp; - struct lro_ctrl *lro = &rxr->lro; - int i, processed = 0, rxdone = 0; - u32 ptype, staterr = 0; - union e1000_adv_rx_desc *cur; - - IGB_RX_LOCK(rxr); - /* Sync the ring. */ - bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - -#ifdef DEV_NETMAP - if (netmap_rx_irq(ifp, rxr->me, &processed)) { - IGB_RX_UNLOCK(rxr); - return (FALSE); - } -#endif /* DEV_NETMAP */ - - /* Main clean loop */ - for (i = rxr->next_to_check; count != 0;) { - struct mbuf *sendmp, *mh, *mp; - struct igb_rx_buf *rxbuf; - u16 hlen, plen, hdr, vtag, pkt_info; - bool eop = FALSE; - - cur = &rxr->rx_base[i]; - staterr = le32toh(cur->wb.upper.status_error); - if ((staterr & E1000_RXD_STAT_DD) == 0) - break; - if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) - break; - count--; - sendmp = mh = mp = NULL; - cur->wb.upper.status_error = 0; - rxbuf = &rxr->rx_buffers[i]; - plen = le16toh(cur->wb.upper.length); - ptype = le32toh(cur->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK; - if (((adapter->hw.mac.type == e1000_i350) || - (adapter->hw.mac.type == e1000_i354)) && - (staterr & E1000_RXDEXT_STATERR_LB)) - vtag = be16toh(cur->wb.upper.vlan); - else - vtag = le16toh(cur->wb.upper.vlan); - hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); - pkt_info = le16toh(cur->wb.lower.lo_dword.hs_rss.pkt_info); - eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP); - - /* - * Free the frame (all segments) if we're at EOP and - * it's an error. - * - * The datasheet states that EOP + status is only valid for - * the final segment in a multi-segment frame. - */ - if (eop && ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0)) { - adapter->dropped_pkts++; - ++rxr->rx_discarded; - igb_rx_discard(rxr, i); - goto next_desc; - } - - /* - ** The way the hardware is configured to - ** split, it will ONLY use the header buffer - ** when header split is enabled, otherwise we - ** get normal behavior, ie, both header and - ** payload are DMA'd into the payload buffer. - ** - ** The fmp test is to catch the case where a - ** packet spans multiple descriptors, in that - ** case only the first header is valid. - */ - if (rxr->hdr_split && rxr->fmp == NULL) { - bus_dmamap_unload(rxr->htag, rxbuf->hmap); - hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >> - E1000_RXDADV_HDRBUFLEN_SHIFT; - if (hlen > IGB_HDR_BUF) - hlen = IGB_HDR_BUF; - mh = rxr->rx_buffers[i].m_head; - mh->m_len = hlen; - /* clear buf pointer for refresh */ - rxbuf->m_head = NULL; - /* - ** Get the payload length, this - ** could be zero if its a small - ** packet. - */ - if (plen > 0) { - mp = rxr->rx_buffers[i].m_pack; - mp->m_len = plen; - mh->m_next = mp; - /* clear buf pointer */ - rxbuf->m_pack = NULL; - rxr->rx_split_packets++; - } - } else { - /* - ** Either no header split, or a - ** secondary piece of a fragmented - ** split packet. - */ - mh = rxr->rx_buffers[i].m_pack; - mh->m_len = plen; - /* clear buf info for refresh */ - rxbuf->m_pack = NULL; - } - bus_dmamap_unload(rxr->ptag, rxbuf->pmap); - - ++processed; /* So we know when to refresh */ - - /* Initial frame - setup */ - if (rxr->fmp == NULL) { - mh->m_pkthdr.len = mh->m_len; - /* Save the head of the chain */ - rxr->fmp = mh; - rxr->lmp = mh; - if (mp != NULL) { - /* Add payload if split */ - mh->m_pkthdr.len += mp->m_len; - rxr->lmp = mh->m_next; - } - } else { - /* Chain mbuf's together */ - rxr->lmp->m_next = mh; - rxr->lmp = rxr->lmp->m_next; - rxr->fmp->m_pkthdr.len += mh->m_len; - } - - if (eop) { - rxr->fmp->m_pkthdr.rcvif = ifp; - rxr->rx_packets++; - /* capture data for AIM */ - rxr->packets++; - rxr->bytes += rxr->fmp->m_pkthdr.len; - rxr->rx_bytes += rxr->fmp->m_pkthdr.len; - - if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) - igb_rx_checksum(staterr, rxr->fmp, ptype); - - if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && - (staterr & E1000_RXD_STAT_VP) != 0) { - rxr->fmp->m_pkthdr.ether_vtag = vtag; - rxr->fmp->m_flags |= M_VLANTAG; - } - - /* - * In case of multiqueue, we have RXCSUM.PCSD bit set - * and never cleared. This means we have RSS hash - * available to be used. - */ - if (adapter->num_queues > 1) { - rxr->fmp->m_pkthdr.flowid = - le32toh(cur->wb.lower.hi_dword.rss); - switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) { - case E1000_RXDADV_RSSTYPE_IPV4_TCP: - M_HASHTYPE_SET(rxr->fmp, - M_HASHTYPE_RSS_TCP_IPV4); - break; - case E1000_RXDADV_RSSTYPE_IPV4: - M_HASHTYPE_SET(rxr->fmp, - M_HASHTYPE_RSS_IPV4); - break; - case E1000_RXDADV_RSSTYPE_IPV6_TCP: - M_HASHTYPE_SET(rxr->fmp, - M_HASHTYPE_RSS_TCP_IPV6); - break; - case E1000_RXDADV_RSSTYPE_IPV6_EX: - M_HASHTYPE_SET(rxr->fmp, - M_HASHTYPE_RSS_IPV6_EX); - break; - case E1000_RXDADV_RSSTYPE_IPV6: - M_HASHTYPE_SET(rxr->fmp, - M_HASHTYPE_RSS_IPV6); - break; - case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX: - M_HASHTYPE_SET(rxr->fmp, - M_HASHTYPE_RSS_TCP_IPV6_EX); - break; - default: - /* XXX fallthrough */ - M_HASHTYPE_SET(rxr->fmp, - M_HASHTYPE_OPAQUE_HASH); - } - } else { -#ifndef IGB_LEGACY_TX - rxr->fmp->m_pkthdr.flowid = que->msix; - M_HASHTYPE_SET(rxr->fmp, M_HASHTYPE_OPAQUE); -#endif - } - sendmp = rxr->fmp; - /* Make sure to set M_PKTHDR. */ - sendmp->m_flags |= M_PKTHDR; - rxr->fmp = NULL; - rxr->lmp = NULL; - } - -next_desc: - bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - - /* Advance our pointers to the next descriptor. */ - if (++i == adapter->num_rx_desc) - i = 0; - /* - ** Send to the stack or LRO - */ - if (sendmp != NULL) { - rxr->next_to_check = i; - igb_rx_input(rxr, ifp, sendmp, ptype); - i = rxr->next_to_check; - rxdone++; - } - - /* Every 8 descriptors we go to refresh mbufs */ - if (processed == 8) { - igb_refresh_mbufs(rxr, i); - processed = 0; - } - } - - /* Catch any remainders */ - if (igb_rx_unrefreshed(rxr)) - igb_refresh_mbufs(rxr, i); - - rxr->next_to_check = i; - - /* - * Flush any outstanding LRO work - */ - tcp_lro_flush_all(lro); - - if (done != NULL) - *done += rxdone; - - IGB_RX_UNLOCK(rxr); - return ((staterr & E1000_RXD_STAT_DD) ? TRUE : FALSE); -} - -/********************************************************************* - * - * Verify that the hardware indicated that the checksum is valid. - * Inform the stack about the status of checksum so that stack - * doesn't spend time verifying the checksum. - * - *********************************************************************/ -static void -igb_rx_checksum(u32 staterr, struct mbuf *mp, u32 ptype) -{ - u16 status = (u16)staterr; - u8 errors = (u8) (staterr >> 24); - int sctp; - - /* Ignore Checksum bit is set */ - if (status & E1000_RXD_STAT_IXSM) { - mp->m_pkthdr.csum_flags = 0; - return; - } - - if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 && - (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0) - sctp = 1; - else - sctp = 0; - if (status & E1000_RXD_STAT_IPCS) { - /* Did it pass? */ - if (!(errors & E1000_RXD_ERR_IPE)) { - /* IP Checksum Good */ - mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; - mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; - } else - mp->m_pkthdr.csum_flags = 0; - } - - if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) { - u64 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); -#if __FreeBSD_version >= 800000 - if (sctp) /* reassign */ - type = CSUM_SCTP_VALID; -#endif - /* Did it pass? */ - if (!(errors & E1000_RXD_ERR_TCPE)) { - mp->m_pkthdr.csum_flags |= type; - if (sctp == 0) - mp->m_pkthdr.csum_data = htons(0xffff); - } - } - return; -} - -/* - * This routine is run via an vlan - * config EVENT - */ -static void -igb_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) -{ - struct adapter *adapter = ifp->if_softc; - u32 index, bit; - - if (ifp->if_softc != arg) /* Not our event */ - return; - - if ((vtag == 0) || (vtag > 4095)) /* Invalid */ - return; - - IGB_CORE_LOCK(adapter); - index = (vtag >> 5) & 0x7F; - bit = vtag & 0x1F; - adapter->shadow_vfta[index] |= (1 << bit); - ++adapter->num_vlans; - /* Change hw filter setting */ - if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) - igb_setup_vlan_hw_support(adapter); - IGB_CORE_UNLOCK(adapter); -} - -/* - * This routine is run via an vlan - * unconfig EVENT - */ -static void -igb_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) -{ - struct adapter *adapter = ifp->if_softc; - u32 index, bit; - - if (ifp->if_softc != arg) - return; - - if ((vtag == 0) || (vtag > 4095)) /* Invalid */ - return; - - IGB_CORE_LOCK(adapter); - index = (vtag >> 5) & 0x7F; - bit = vtag & 0x1F; - adapter->shadow_vfta[index] &= ~(1 << bit); - --adapter->num_vlans; - /* Change hw filter setting */ - if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) - igb_setup_vlan_hw_support(adapter); - IGB_CORE_UNLOCK(adapter); -} - -static void -igb_setup_vlan_hw_support(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - struct ifnet *ifp = adapter->ifp; - u32 reg; - - if (adapter->vf_ifp) { - e1000_rlpml_set_vf(hw, - adapter->max_frame_size + VLAN_TAG_SIZE); - return; - } - - reg = E1000_READ_REG(hw, E1000_CTRL); - reg |= E1000_CTRL_VME; - E1000_WRITE_REG(hw, E1000_CTRL, reg); - - /* Enable the Filter Table */ - if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { - reg = E1000_READ_REG(hw, E1000_RCTL); - reg &= ~E1000_RCTL_CFIEN; - reg |= E1000_RCTL_VFE; - E1000_WRITE_REG(hw, E1000_RCTL, reg); - } - - /* Update the frame size */ - E1000_WRITE_REG(&adapter->hw, E1000_RLPML, - adapter->max_frame_size + VLAN_TAG_SIZE); - - /* Don't bother with table if no vlans */ - if ((adapter->num_vlans == 0) || - ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)) - return; - /* - ** A soft reset zero's out the VFTA, so - ** we need to repopulate it now. - */ - for (int i = 0; i < IGB_VFTA_SIZE; i++) - if (adapter->shadow_vfta[i] != 0) { - if (adapter->vf_ifp) - e1000_vfta_set_vf(hw, - adapter->shadow_vfta[i], TRUE); - else - e1000_write_vfta(hw, - i, adapter->shadow_vfta[i]); - } -} - -static void -igb_enable_intr(struct adapter *adapter) -{ - /* With RSS set up what to auto clear */ - if (adapter->msix_mem) { - u32 mask = (adapter->que_mask | adapter->link_mask); - E1000_WRITE_REG(&adapter->hw, E1000_EIAC, mask); - E1000_WRITE_REG(&adapter->hw, E1000_EIAM, mask); - E1000_WRITE_REG(&adapter->hw, E1000_EIMS, mask); - E1000_WRITE_REG(&adapter->hw, E1000_IMS, - E1000_IMS_LSC); - } else { - E1000_WRITE_REG(&adapter->hw, E1000_IMS, - IMS_ENABLE_MASK); - } - E1000_WRITE_FLUSH(&adapter->hw); - - return; -} - -static void -igb_disable_intr(struct adapter *adapter) -{ - if (adapter->msix_mem) { - E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0); - E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0); - } - E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0); - E1000_WRITE_FLUSH(&adapter->hw); - return; -} - -/* - * Bit of a misnomer, what this really means is - * to enable OS management of the system... aka - * to disable special hardware management features - */ -static void -igb_init_manageability(struct adapter *adapter) -{ - if (adapter->has_manage) { - int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H); - int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); - - /* disable hardware interception of ARP */ - manc &= ~(E1000_MANC_ARP_EN); - - /* enable receiving management packets to the host */ - manc |= E1000_MANC_EN_MNG2HOST; - manc2h |= 1 << 5; /* Mng Port 623 */ - manc2h |= 1 << 6; /* Mng Port 664 */ - E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h); - E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); - } -} - -/* - * Give control back to hardware management - * controller if there is one. - */ -static void -igb_release_manageability(struct adapter *adapter) -{ - if (adapter->has_manage) { - int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); - - /* re-enable hardware interception of ARP */ - manc |= E1000_MANC_ARP_EN; - manc &= ~E1000_MANC_EN_MNG2HOST; - - E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); - } -} - -/* - * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit. - * For ASF and Pass Through versions of f/w this means that - * the driver is loaded. - * - */ -static void -igb_get_hw_control(struct adapter *adapter) -{ - u32 ctrl_ext; - - if (adapter->vf_ifp) - return; - - /* Let firmware know the driver has taken over */ - ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); - E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, - ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); -} - -/* - * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit. - * For ASF and Pass Through versions of f/w this means that the - * driver is no longer loaded. - * - */ -static void -igb_release_hw_control(struct adapter *adapter) -{ - u32 ctrl_ext; - - if (adapter->vf_ifp) - return; - - /* Let firmware taken over control of h/w */ - ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); - E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, - ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); -} - -static int -igb_is_valid_ether_addr(uint8_t *addr) -{ - char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; - - if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { - return (FALSE); - } - - return (TRUE); -} - - -/* - * Enable PCI Wake On Lan capability - */ -static void -igb_enable_wakeup(device_t dev) -{ - u16 cap, status; - u8 id; - - /* First find the capabilities pointer*/ - cap = pci_read_config(dev, PCIR_CAP_PTR, 2); - /* Read the PM Capabilities */ - id = pci_read_config(dev, cap, 1); - if (id != PCIY_PMG) /* Something wrong */ - return; - /* OK, we have the power capabilities, so - now get the status register */ - cap += PCIR_POWER_STATUS; - status = pci_read_config(dev, cap, 2); - status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; - pci_write_config(dev, cap, status, 2); - return; -} - -static void -igb_led_func(void *arg, int onoff) -{ - struct adapter *adapter = arg; - - IGB_CORE_LOCK(adapter); - if (onoff) { - e1000_setup_led(&adapter->hw); - e1000_led_on(&adapter->hw); - } else { - e1000_led_off(&adapter->hw); - e1000_cleanup_led(&adapter->hw); - } - IGB_CORE_UNLOCK(adapter); -} - -static uint64_t -igb_get_vf_counter(if_t ifp, ift_counter cnt) -{ - struct adapter *adapter; - struct e1000_vf_stats *stats; -#ifndef IGB_LEGACY_TX - struct tx_ring *txr; - uint64_t rv; -#endif - - adapter = if_getsoftc(ifp); - stats = (struct e1000_vf_stats *)adapter->stats; - - switch (cnt) { - case IFCOUNTER_IPACKETS: - return (stats->gprc); - case IFCOUNTER_OPACKETS: - return (stats->gptc); - case IFCOUNTER_IBYTES: - return (stats->gorc); - case IFCOUNTER_OBYTES: - return (stats->gotc); - case IFCOUNTER_IMCASTS: - return (stats->mprc); - case IFCOUNTER_IERRORS: - return (adapter->dropped_pkts); - case IFCOUNTER_OERRORS: - return (adapter->watchdog_events); -#ifndef IGB_LEGACY_TX - case IFCOUNTER_OQDROPS: - rv = 0; - txr = adapter->tx_rings; - for (int i = 0; i < adapter->num_queues; i++, txr++) - rv += txr->br->br_drops; - return (rv); -#endif - default: - return (if_get_counter_default(ifp, cnt)); - } -} - -static uint64_t -igb_get_counter(if_t ifp, ift_counter cnt) -{ - struct adapter *adapter; - struct e1000_hw_stats *stats; -#ifndef IGB_LEGACY_TX - struct tx_ring *txr; - uint64_t rv; -#endif - - adapter = if_getsoftc(ifp); - if (adapter->vf_ifp) - return (igb_get_vf_counter(ifp, cnt)); - - stats = (struct e1000_hw_stats *)adapter->stats; - - switch (cnt) { - case IFCOUNTER_IPACKETS: - return (stats->gprc); - case IFCOUNTER_OPACKETS: - return (stats->gptc); - case IFCOUNTER_IBYTES: - return (stats->gorc); - case IFCOUNTER_OBYTES: - return (stats->gotc); - case IFCOUNTER_IMCASTS: - return (stats->mprc); - case IFCOUNTER_OMCASTS: - return (stats->mptc); - case IFCOUNTER_IERRORS: - return (adapter->dropped_pkts + stats->rxerrc + - stats->crcerrs + stats->algnerrc + - stats->ruc + stats->roc + stats->cexterr); - case IFCOUNTER_OERRORS: - return (stats->ecol + stats->latecol + - adapter->watchdog_events); - case IFCOUNTER_COLLISIONS: - return (stats->colc); - case IFCOUNTER_IQDROPS: - return (stats->mpc); -#ifndef IGB_LEGACY_TX - case IFCOUNTER_OQDROPS: - rv = 0; - txr = adapter->tx_rings; - for (int i = 0; i < adapter->num_queues; i++, txr++) - rv += txr->br->br_drops; - return (rv); -#endif - default: - return (if_get_counter_default(ifp, cnt)); - } -} - -/********************************************************************** - * - * Update the board statistics counters. - * - **********************************************************************/ -static void -igb_update_stats_counters(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - struct e1000_hw_stats *stats; - - /* - ** The virtual function adapter has only a - ** small controlled set of stats, do only - ** those and return. - */ - if (adapter->vf_ifp) { - igb_update_vf_stats_counters(adapter); - return; - } - - stats = (struct e1000_hw_stats *)adapter->stats; - - if (adapter->hw.phy.media_type == e1000_media_type_copper || - (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) { - stats->symerrs += - E1000_READ_REG(hw,E1000_SYMERRS); - stats->sec += E1000_READ_REG(hw, E1000_SEC); - } - - stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS); - stats->mpc += E1000_READ_REG(hw, E1000_MPC); - stats->scc += E1000_READ_REG(hw, E1000_SCC); - stats->ecol += E1000_READ_REG(hw, E1000_ECOL); - - stats->mcc += E1000_READ_REG(hw, E1000_MCC); - stats->latecol += E1000_READ_REG(hw, E1000_LATECOL); - stats->colc += E1000_READ_REG(hw, E1000_COLC); - stats->dc += E1000_READ_REG(hw, E1000_DC); - stats->rlec += E1000_READ_REG(hw, E1000_RLEC); - stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC); - stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC); - /* - ** For watchdog management we need to know if we have been - ** paused during the last interval, so capture that here. - */ - adapter->pause_frames = E1000_READ_REG(&adapter->hw, E1000_XOFFRXC); - stats->xoffrxc += adapter->pause_frames; - stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC); - stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC); - stats->prc64 += E1000_READ_REG(hw, E1000_PRC64); - stats->prc127 += E1000_READ_REG(hw, E1000_PRC127); - stats->prc255 += E1000_READ_REG(hw, E1000_PRC255); - stats->prc511 += E1000_READ_REG(hw, E1000_PRC511); - stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023); - stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522); - stats->gprc += E1000_READ_REG(hw, E1000_GPRC); - stats->bprc += E1000_READ_REG(hw, E1000_BPRC); - stats->mprc += E1000_READ_REG(hw, E1000_MPRC); - stats->gptc += E1000_READ_REG(hw, E1000_GPTC); - - /* For the 64-bit byte counters the low dword must be read first. */ - /* Both registers clear on the read of the high dword */ - - stats->gorc += E1000_READ_REG(hw, E1000_GORCL) + - ((u64)E1000_READ_REG(hw, E1000_GORCH) << 32); - stats->gotc += E1000_READ_REG(hw, E1000_GOTCL) + - ((u64)E1000_READ_REG(hw, E1000_GOTCH) << 32); - - stats->rnbc += E1000_READ_REG(hw, E1000_RNBC); - stats->ruc += E1000_READ_REG(hw, E1000_RUC); - stats->rfc += E1000_READ_REG(hw, E1000_RFC); - stats->roc += E1000_READ_REG(hw, E1000_ROC); - stats->rjc += E1000_READ_REG(hw, E1000_RJC); - - stats->mgprc += E1000_READ_REG(hw, E1000_MGTPRC); - stats->mgpdc += E1000_READ_REG(hw, E1000_MGTPDC); - stats->mgptc += E1000_READ_REG(hw, E1000_MGTPTC); - - stats->tor += E1000_READ_REG(hw, E1000_TORL) + - ((u64)E1000_READ_REG(hw, E1000_TORH) << 32); - stats->tot += E1000_READ_REG(hw, E1000_TOTL) + - ((u64)E1000_READ_REG(hw, E1000_TOTH) << 32); - - stats->tpr += E1000_READ_REG(hw, E1000_TPR); - stats->tpt += E1000_READ_REG(hw, E1000_TPT); - stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64); - stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127); - stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255); - stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511); - stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023); - stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522); - stats->mptc += E1000_READ_REG(hw, E1000_MPTC); - stats->bptc += E1000_READ_REG(hw, E1000_BPTC); - - /* Interrupt Counts */ - - stats->iac += E1000_READ_REG(hw, E1000_IAC); - stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC); - stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC); - stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC); - stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC); - stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC); - stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC); - stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC); - stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC); - - /* Host to Card Statistics */ - - stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC); - stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC); - stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC); - stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC); - stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC); - stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC); - stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC); - stats->hgorc += (E1000_READ_REG(hw, E1000_HGORCL) + - ((u64)E1000_READ_REG(hw, E1000_HGORCH) << 32)); - stats->hgotc += (E1000_READ_REG(hw, E1000_HGOTCL) + - ((u64)E1000_READ_REG(hw, E1000_HGOTCH) << 32)); - stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS); - stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC); - stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC); - - stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC); - stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC); - stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS); - stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR); - stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC); - stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC); - - /* Driver specific counters */ - adapter->device_control = E1000_READ_REG(hw, E1000_CTRL); - adapter->rx_control = E1000_READ_REG(hw, E1000_RCTL); - adapter->int_mask = E1000_READ_REG(hw, E1000_IMS); - adapter->eint_mask = E1000_READ_REG(hw, E1000_EIMS); - adapter->packet_buf_alloc_tx = - ((E1000_READ_REG(hw, E1000_PBA) & 0xffff0000) >> 16); - adapter->packet_buf_alloc_rx = - (E1000_READ_REG(hw, E1000_PBA) & 0xffff); -} - - -/********************************************************************** - * - * Initialize the VF board statistics counters. - * - **********************************************************************/ -static void -igb_vf_init_stats(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - struct e1000_vf_stats *stats; - - stats = (struct e1000_vf_stats *)adapter->stats; - if (stats == NULL) - return; - stats->last_gprc = E1000_READ_REG(hw, E1000_VFGPRC); - stats->last_gorc = E1000_READ_REG(hw, E1000_VFGORC); - stats->last_gptc = E1000_READ_REG(hw, E1000_VFGPTC); - stats->last_gotc = E1000_READ_REG(hw, E1000_VFGOTC); - stats->last_mprc = E1000_READ_REG(hw, E1000_VFMPRC); -} - -/********************************************************************** - * - * Update the VF board statistics counters. - * - **********************************************************************/ -static void -igb_update_vf_stats_counters(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - struct e1000_vf_stats *stats; - - if (adapter->link_speed == 0) - return; - - stats = (struct e1000_vf_stats *)adapter->stats; - - UPDATE_VF_REG(E1000_VFGPRC, - stats->last_gprc, stats->gprc); - UPDATE_VF_REG(E1000_VFGORC, - stats->last_gorc, stats->gorc); - UPDATE_VF_REG(E1000_VFGPTC, - stats->last_gptc, stats->gptc); - UPDATE_VF_REG(E1000_VFGOTC, - stats->last_gotc, stats->gotc); - UPDATE_VF_REG(E1000_VFMPRC, - stats->last_mprc, stats->mprc); -} - -/* Export a single 32-bit register via a read-only sysctl. */ -static int -igb_sysctl_reg_handler(SYSCTL_HANDLER_ARGS) -{ - struct adapter *adapter; - u_int val; - - adapter = oidp->oid_arg1; - val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2); - return (sysctl_handle_int(oidp, &val, 0, req)); -} - -/* -** Tuneable interrupt rate handler -*/ -static int -igb_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS) -{ - struct igb_queue *que = ((struct igb_queue *)oidp->oid_arg1); - int error; - u32 reg, usec, rate; - - reg = E1000_READ_REG(&que->adapter->hw, E1000_EITR(que->msix)); - usec = ((reg & 0x7FFC) >> 2); - if (usec > 0) - rate = 1000000 / usec; - else - rate = 0; - error = sysctl_handle_int(oidp, &rate, 0, req); - if (error || !req->newptr) - return error; - return 0; -} - -/* - * Add sysctl variables, one per statistic, to the system. - */ -static void -igb_add_hw_stats(struct adapter *adapter) -{ - device_t dev = adapter->dev; - - struct tx_ring *txr = adapter->tx_rings; - struct rx_ring *rxr = adapter->rx_rings; - - struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); - struct sysctl_oid *tree = device_get_sysctl_tree(dev); - struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); - struct e1000_hw_stats *stats = adapter->stats; - - struct sysctl_oid *stat_node, *queue_node, *int_node, *host_node; - struct sysctl_oid_list *stat_list, *queue_list, *int_list, *host_list; - -#define QUEUE_NAME_LEN 32 - char namebuf[QUEUE_NAME_LEN]; - - /* Driver Statistics */ - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", - CTLFLAG_RD, &adapter->dropped_pkts, - "Driver dropped packets"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq", - CTLFLAG_RD, &adapter->link_irq, - "Link MSIX IRQ Handled"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_fail", - CTLFLAG_RD, &adapter->mbuf_defrag_failed, - "Defragmenting mbuf chain failed"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail", - CTLFLAG_RD, &adapter->no_tx_dma_setup, - "Driver tx dma failure in xmit"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns", - CTLFLAG_RD, &adapter->rx_overruns, - "RX overruns"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts", - CTLFLAG_RD, &adapter->watchdog_events, - "Watchdog timeouts"); - - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "device_control", - CTLFLAG_RD, &adapter->device_control, - "Device Control Register"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_control", - CTLFLAG_RD, &adapter->rx_control, - "Receiver Control Register"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "interrupt_mask", - CTLFLAG_RD, &adapter->int_mask, - "Interrupt Mask"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "extended_int_mask", - CTLFLAG_RD, &adapter->eint_mask, - "Extended Interrupt Mask"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_buf_alloc", - CTLFLAG_RD, &adapter->packet_buf_alloc_tx, - "Transmit Buffer Packet Allocation"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_buf_alloc", - CTLFLAG_RD, &adapter->packet_buf_alloc_rx, - "Receive Buffer Packet Allocation"); - SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water", - CTLFLAG_RD, &adapter->hw.fc.high_water, 0, - "Flow Control High Watermark"); - SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", - CTLFLAG_RD, &adapter->hw.fc.low_water, 0, - "Flow Control Low Watermark"); - - for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { - struct lro_ctrl *lro = &rxr->lro; - - snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i); - queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, - CTLFLAG_RD, NULL, "Queue Name"); - queue_list = SYSCTL_CHILDREN(queue_node); - - SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate", - CTLTYPE_UINT | CTLFLAG_RD, &adapter->queues[i], - sizeof(&adapter->queues[i]), - igb_sysctl_interrupt_rate_handler, - "IU", "Interrupt Rate"); - - SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head", - CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDH(txr->me), - igb_sysctl_reg_handler, "IU", - "Transmit Descriptor Head"); - SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail", - CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDT(txr->me), - igb_sysctl_reg_handler, "IU", - "Transmit Descriptor Tail"); - SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "no_desc_avail", - CTLFLAG_RD, &txr->no_desc_avail, - "Queue Descriptors Unavailable"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets", - CTLFLAG_RD, &txr->total_packets, - "Queue Packets Transmitted"); - - SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head", - CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDH(rxr->me), - igb_sysctl_reg_handler, "IU", - "Receive Descriptor Head"); - SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail", - CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDT(rxr->me), - igb_sysctl_reg_handler, "IU", - "Receive Descriptor Tail"); - SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_packets", - CTLFLAG_RD, &rxr->rx_packets, - "Queue Packets Received"); - SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_bytes", - CTLFLAG_RD, &rxr->rx_bytes, - "Queue Bytes Received"); - SYSCTL_ADD_U64(ctx, queue_list, OID_AUTO, "lro_queued", - CTLFLAG_RD, &lro->lro_queued, 0, - "LRO Queued"); - SYSCTL_ADD_U64(ctx, queue_list, OID_AUTO, "lro_flushed", - CTLFLAG_RD, &lro->lro_flushed, 0, - "LRO Flushed"); - } - - /* MAC stats get their own sub node */ - - stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", - CTLFLAG_RD, NULL, "MAC Statistics"); - stat_list = SYSCTL_CHILDREN(stat_node); - - /* - ** VF adapter has a very limited set of stats - ** since its not managing the metal, so to speak. - */ - if (adapter->vf_ifp) { - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd", - CTLFLAG_RD, &stats->gprc, - "Good Packets Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", - CTLFLAG_RD, &stats->gptc, - "Good Packets Transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", - CTLFLAG_RD, &stats->gorc, - "Good Octets Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", - CTLFLAG_RD, &stats->gotc, - "Good Octets Transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd", - CTLFLAG_RD, &stats->mprc, - "Multicast Packets Received"); - return; - } - - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "excess_coll", - CTLFLAG_RD, &stats->ecol, - "Excessive collisions"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "single_coll", - CTLFLAG_RD, &stats->scc, - "Single collisions"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "multiple_coll", - CTLFLAG_RD, &stats->mcc, - "Multiple collisions"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "late_coll", - CTLFLAG_RD, &stats->latecol, - "Late collisions"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "collision_count", - CTLFLAG_RD, &stats->colc, - "Collision Count"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "symbol_errors", - CTLFLAG_RD, &stats->symerrs, - "Symbol Errors"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "sequence_errors", - CTLFLAG_RD, &stats->sec, - "Sequence Errors"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "defer_count", - CTLFLAG_RD, &stats->dc, - "Defer Count"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "missed_packets", - CTLFLAG_RD, &stats->mpc, - "Missed Packets"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_length_errors", - CTLFLAG_RD, &stats->rlec, - "Receive Length Errors"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_no_buff", - CTLFLAG_RD, &stats->rnbc, - "Receive No Buffers"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_undersize", - CTLFLAG_RD, &stats->ruc, - "Receive Undersize"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_fragmented", - CTLFLAG_RD, &stats->rfc, - "Fragmented Packets Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_oversize", - CTLFLAG_RD, &stats->roc, - "Oversized Packets Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_jabber", - CTLFLAG_RD, &stats->rjc, - "Recevied Jabber"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_errs", - CTLFLAG_RD, &stats->rxerrc, - "Receive Errors"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "crc_errs", - CTLFLAG_RD, &stats->crcerrs, - "CRC errors"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "alignment_errs", - CTLFLAG_RD, &stats->algnerrc, - "Alignment Errors"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_no_crs", - CTLFLAG_RD, &stats->tncrs, - "Transmit with No CRS"); - /* On 82575 these are collision counts */ - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs", - CTLFLAG_RD, &stats->cexterr, - "Collision/Carrier extension errors"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_recvd", - CTLFLAG_RD, &stats->xonrxc, - "XON Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_txd", - CTLFLAG_RD, &stats->xontxc, - "XON Transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_recvd", - CTLFLAG_RD, &stats->xoffrxc, - "XOFF Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_txd", - CTLFLAG_RD, &stats->xofftxc, - "XOFF Transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "unsupported_fc_recvd", - CTLFLAG_RD, &stats->fcruc, - "Unsupported Flow Control Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_recvd", - CTLFLAG_RD, &stats->mgprc, - "Management Packets Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_drop", - CTLFLAG_RD, &stats->mgpdc, - "Management Packets Dropped"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_txd", - CTLFLAG_RD, &stats->mgptc, - "Management Packets Transmitted"); - /* Packet Reception Stats */ - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd", - CTLFLAG_RD, &stats->tpr, - "Total Packets Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd", - CTLFLAG_RD, &stats->gprc, - "Good Packets Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd", - CTLFLAG_RD, &stats->bprc, - "Broadcast Packets Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd", - CTLFLAG_RD, &stats->mprc, - "Multicast Packets Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_64", - CTLFLAG_RD, &stats->prc64, - "64 byte frames received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127", - CTLFLAG_RD, &stats->prc127, - "65-127 byte frames received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255", - CTLFLAG_RD, &stats->prc255, - "128-255 byte frames received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511", - CTLFLAG_RD, &stats->prc511, - "256-511 byte frames received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023", - CTLFLAG_RD, &stats->prc1023, - "512-1023 byte frames received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522", - CTLFLAG_RD, &stats->prc1522, - "1023-1522 byte frames received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", - CTLFLAG_RD, &stats->gorc, - "Good Octets Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_recvd", - CTLFLAG_RD, &stats->tor, - "Total Octets Received"); - - /* Packet Transmission Stats */ - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", - CTLFLAG_RD, &stats->gotc, - "Good Octets Transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_txd", - CTLFLAG_RD, &stats->tot, - "Total Octets Transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd", - CTLFLAG_RD, &stats->tpt, - "Total Packets Transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", - CTLFLAG_RD, &stats->gptc, - "Good Packets Transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd", - CTLFLAG_RD, &stats->bptc, - "Broadcast Packets Transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd", - CTLFLAG_RD, &stats->mptc, - "Multicast Packets Transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_64", - CTLFLAG_RD, &stats->ptc64, - "64 byte frames transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127", - CTLFLAG_RD, &stats->ptc127, - "65-127 byte frames transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255", - CTLFLAG_RD, &stats->ptc255, - "128-255 byte frames transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511", - CTLFLAG_RD, &stats->ptc511, - "256-511 byte frames transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023", - CTLFLAG_RD, &stats->ptc1023, - "512-1023 byte frames transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522", - CTLFLAG_RD, &stats->ptc1522, - "1024-1522 byte frames transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_txd", - CTLFLAG_RD, &stats->tsctc, - "TSO Contexts Transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail", - CTLFLAG_RD, &stats->tsctfc, - "TSO Contexts Failed"); - - - /* Interrupt Stats */ - - int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts", - CTLFLAG_RD, NULL, "Interrupt Statistics"); - int_list = SYSCTL_CHILDREN(int_node); - - SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "asserts", - CTLFLAG_RD, &stats->iac, - "Interrupt Assertion Count"); - - SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer", - CTLFLAG_RD, &stats->icrxptc, - "Interrupt Cause Rx Pkt Timer Expire Count"); - - SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_abs_timer", - CTLFLAG_RD, &stats->icrxatc, - "Interrupt Cause Rx Abs Timer Expire Count"); - - SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer", - CTLFLAG_RD, &stats->ictxptc, - "Interrupt Cause Tx Pkt Timer Expire Count"); - - SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_abs_timer", - CTLFLAG_RD, &stats->ictxatc, - "Interrupt Cause Tx Abs Timer Expire Count"); - - SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_empty", - CTLFLAG_RD, &stats->ictxqec, - "Interrupt Cause Tx Queue Empty Count"); - - SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh", - CTLFLAG_RD, &stats->ictxqmtc, - "Interrupt Cause Tx Queue Min Thresh Count"); - - SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh", - CTLFLAG_RD, &stats->icrxdmtc, - "Interrupt Cause Rx Desc Min Thresh Count"); - - SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_overrun", - CTLFLAG_RD, &stats->icrxoc, - "Interrupt Cause Receiver Overrun Count"); - - /* Host to Card Stats */ - - host_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "host", - CTLFLAG_RD, NULL, - "Host to Card Statistics"); - - host_list = SYSCTL_CHILDREN(host_node); - - SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt", - CTLFLAG_RD, &stats->cbtmpc, - "Circuit Breaker Tx Packet Count"); - - SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "host_tx_pkt_discard", - CTLFLAG_RD, &stats->htdpmc, - "Host Transmit Discarded Packets"); - - SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_pkt", - CTLFLAG_RD, &stats->rpthc, - "Rx Packets To Host"); - - SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkts", - CTLFLAG_RD, &stats->cbrmpc, - "Circuit Breaker Rx Packet Count"); - - SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkt_drop", - CTLFLAG_RD, &stats->cbrdpc, - "Circuit Breaker Rx Dropped Count"); - - SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_pkt", - CTLFLAG_RD, &stats->hgptc, - "Host Good Packets Tx Count"); - - SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt_drop", - CTLFLAG_RD, &stats->htcbdpc, - "Host Tx Circuit Breaker Dropped Count"); - - SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_good_bytes", - CTLFLAG_RD, &stats->hgorc, - "Host Good Octets Received Count"); - - SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_bytes", - CTLFLAG_RD, &stats->hgotc, - "Host Good Octets Transmit Count"); - - SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "length_errors", - CTLFLAG_RD, &stats->lenerrs, - "Length Errors"); - - SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "serdes_violation_pkt", - CTLFLAG_RD, &stats->scvpc, - "SerDes/SGMII Code Violation Pkt Count"); - - SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "header_redir_missed", - CTLFLAG_RD, &stats->hrmpc, - "Header Redirection Missed Packet Count"); -} - - -/********************************************************************** - * - * This routine provides a way to dump out the adapter eeprom, - * often a useful debug/service tool. This only dumps the first - * 32 words, stuff that matters is in that extent. - * - **********************************************************************/ -static int -igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS) -{ - struct adapter *adapter; - int error; - int result; - - result = -1; - error = sysctl_handle_int(oidp, &result, 0, req); - - if (error || !req->newptr) - return (error); - - /* - * This value will cause a hex dump of the - * first 32 16-bit words of the EEPROM to - * the screen. - */ - if (result == 1) { - adapter = (struct adapter *)arg1; - igb_print_nvm_info(adapter); - } - - return (error); -} - -static void -igb_print_nvm_info(struct adapter *adapter) -{ - u16 eeprom_data; - int i, j, row = 0; - - /* Its a bit crude, but it gets the job done */ - printf("\nInterface EEPROM Dump:\n"); - printf("Offset\n0x0000 "); - for (i = 0, j = 0; i < 32; i++, j++) { - if (j == 8) { /* Make the offset block */ - j = 0; ++row; - printf("\n0x00%x0 ",row); - } - e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data); - printf("%04x ", eeprom_data); - } - printf("\n"); -} - -static void -igb_set_sysctl_value(struct adapter *adapter, const char *name, - const char *description, int *limit, int value) -{ - *limit = value; - SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), - OID_AUTO, name, CTLFLAG_RW, limit, value, description); -} - -/* -** Set flow control using sysctl: -** Flow control values: -** 0 - off -** 1 - rx pause -** 2 - tx pause -** 3 - full -*/ -static int -igb_set_flowcntl(SYSCTL_HANDLER_ARGS) -{ - int error; - static int input = 3; /* default is full */ - struct adapter *adapter = (struct adapter *) arg1; - - error = sysctl_handle_int(oidp, &input, 0, req); - - if ((error) || (req->newptr == NULL)) - return (error); - - switch (input) { - case e1000_fc_rx_pause: - case e1000_fc_tx_pause: - case e1000_fc_full: - case e1000_fc_none: - adapter->hw.fc.requested_mode = input; - adapter->fc = input; - break; - default: - /* Do nothing */ - return (error); - } - - adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode; - e1000_force_mac_fc(&adapter->hw); - /* XXX TODO: update DROP_EN on each RX queue if appropriate */ - return (error); -} - -/* -** Manage DMA Coalesce: -** Control values: -** 0/1 - off/on -** Legal timer values are: -** 250,500,1000-10000 in thousands -*/ -static int -igb_sysctl_dmac(SYSCTL_HANDLER_ARGS) -{ - struct adapter *adapter = (struct adapter *) arg1; - int error; - - error = sysctl_handle_int(oidp, &adapter->dmac, 0, req); - - if ((error) || (req->newptr == NULL)) - return (error); - - switch (adapter->dmac) { - case 0: - /* Disabling */ - break; - case 1: /* Just enable and use default */ - adapter->dmac = 1000; - break; - case 250: - case 500: - case 1000: - case 2000: - case 3000: - case 4000: - case 5000: - case 6000: - case 7000: - case 8000: - case 9000: - case 10000: - /* Legal values - allow */ - break; - default: - /* Do nothing, illegal value */ - adapter->dmac = 0; - return (EINVAL); - } - /* Reinit the interface */ - igb_init(adapter); - return (error); -} - -/* -** Manage Energy Efficient Ethernet: -** Control values: -** 0/1 - enabled/disabled -*/ -static int -igb_sysctl_eee(SYSCTL_HANDLER_ARGS) -{ - struct adapter *adapter = (struct adapter *) arg1; - int error, value; - - value = adapter->hw.dev_spec._82575.eee_disable; - error = sysctl_handle_int(oidp, &value, 0, req); - if (error || req->newptr == NULL) - return (error); - IGB_CORE_LOCK(adapter); - adapter->hw.dev_spec._82575.eee_disable = (value != 0); - igb_init_locked(adapter); - IGB_CORE_UNLOCK(adapter); - return (0); -} diff --git a/freebsd/sys/dev/e1000/if_igb.h b/freebsd/sys/dev/e1000/if_igb.h deleted file mode 100644 index ea5ba649..00000000 --- a/freebsd/sys/dev/e1000/if_igb.h +++ /dev/null @@ -1,634 +0,0 @@ -/****************************************************************************** - - Copyright (c) 2001-2015, Intel Corporation - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - -******************************************************************************/ -/*$FreeBSD$*/ - -#ifndef _IF_IGB_H_ -#define _IF_IGB_H_ - -#ifdef ALTQ -#define IGB_LEGACY_TX -#endif - -#include <rtems/bsd/sys/param.h> -#include <sys/systm.h> -#ifndef IGB_LEGACY_TX -#include <sys/buf_ring.h> -#endif -#include <sys/bus.h> -#include <sys/endian.h> -#include <sys/kernel.h> -#include <sys/kthread.h> -#include <sys/malloc.h> -#include <sys/mbuf.h> -#include <sys/module.h> -#include <sys/rman.h> -#include <sys/socket.h> -#include <sys/sockio.h> -#include <sys/sysctl.h> -#include <sys/taskqueue.h> -#include <sys/eventhandler.h> -#include <sys/pcpu.h> -#include <sys/smp.h> -#include <machine/smp.h> -#include <machine/bus.h> -#include <machine/resource.h> - -#include <net/bpf.h> -#include <net/ethernet.h> -#include <net/if.h> -#include <net/if_var.h> -#include <net/if_arp.h> -#include <net/if_dl.h> -#include <net/if_media.h> -#ifdef RSS -#include <net/rss_config.h> -#include <netinet/in_rss.h> -#endif - -#include <net/if_types.h> -#include <net/if_vlan_var.h> - -#include <netinet/in_systm.h> -#include <netinet/in.h> -#include <netinet/if_ether.h> -#include <netinet/ip.h> -#include <netinet/ip6.h> -#include <netinet/tcp.h> -#include <netinet/tcp_lro.h> -#include <netinet/udp.h> - -#include <machine/in_cksum.h> -#include <dev/led/led.h> -#include <dev/pci/pcivar.h> -#include <dev/pci/pcireg.h> - -#include "e1000_api.h" -#include "e1000_82575.h" - -/* Tunables */ -/* - * IGB_TXD: Maximum number of Transmit Descriptors - * - * This value is the number of transmit descriptors allocated by the driver. - * Increasing this value allows the driver to queue more transmits. Each - * descriptor is 16 bytes. - * Since TDLEN should be multiple of 128bytes, the number of transmit - * desscriptors should meet the following condition. - * (num_tx_desc * sizeof(struct e1000_tx_desc)) % 128 == 0 - */ -#define IGB_MIN_TXD 256 -#define IGB_DEFAULT_TXD 1024 -#define IGB_MAX_TXD 4096 - -/* - * IGB_RXD: Maximum number of Receive Descriptors - * - * This value is the number of receive descriptors allocated by the driver. - * Increasing this value allows the driver to buffer more incoming packets. - * Each descriptor is 16 bytes. A receive buffer is also allocated for each - * descriptor. The maximum MTU size is 16110. - * Since TDLEN should be multiple of 128bytes, the number of transmit - * desscriptors should meet the following condition. - * (num_tx_desc * sizeof(struct e1000_tx_desc)) % 128 == 0 - */ -#define IGB_MIN_RXD 256 -#define IGB_DEFAULT_RXD 1024 -#define IGB_MAX_RXD 4096 - -/* - * IGB_TIDV - Transmit Interrupt Delay Value - * Valid Range: 0-65535 (0=off) - * Default Value: 64 - * This value delays the generation of transmit interrupts in units of - * 1.024 microseconds. Transmit interrupt reduction can improve CPU - * efficiency if properly tuned for specific network traffic. If the - * system is reporting dropped transmits, this value may be set too high - * causing the driver to run out of available transmit descriptors. - */ -#define IGB_TIDV 64 - -/* - * IGB_TADV - Transmit Absolute Interrupt Delay Value - * Valid Range: 0-65535 (0=off) - * Default Value: 64 - * This value, in units of 1.024 microseconds, limits the delay in which a - * transmit interrupt is generated. Useful only if IGB_TIDV is non-zero, - * this value ensures that an interrupt is generated after the initial - * packet is sent on the wire within the set amount of time. Proper tuning, - * along with IGB_TIDV, may improve traffic throughput in specific - * network conditions. - */ -#define IGB_TADV 64 - -/* - * IGB_RDTR - Receive Interrupt Delay Timer (Packet Timer) - * Valid Range: 0-65535 (0=off) - * Default Value: 0 - * This value delays the generation of receive interrupts in units of 1.024 - * microseconds. Receive interrupt reduction can improve CPU efficiency if - * properly tuned for specific network traffic. Increasing this value adds - * extra latency to frame reception and can end up decreasing the throughput - * of TCP traffic. If the system is reporting dropped receives, this value - * may be set too high, causing the driver to run out of available receive - * descriptors. - * - * CAUTION: When setting IGB_RDTR to a value other than 0, adapters - * may hang (stop transmitting) under certain network conditions. - * If this occurs a WATCHDOG message is logged in the system - * event log. In addition, the controller is automatically reset, - * restoring the network connection. To eliminate the potential - * for the hang ensure that IGB_RDTR is set to 0. - */ -#define IGB_RDTR 0 - -/* - * Receive Interrupt Absolute Delay Timer (Not valid for 82542/82543/82544) - * Valid Range: 0-65535 (0=off) - * Default Value: 64 - * This value, in units of 1.024 microseconds, limits the delay in which a - * receive interrupt is generated. Useful only if IGB_RDTR is non-zero, - * this value ensures that an interrupt is generated after the initial - * packet is received within the set amount of time. Proper tuning, - * along with IGB_RDTR, may improve traffic throughput in specific network - * conditions. - */ -#define IGB_RADV 64 - -/* - * This parameter controls the duration of transmit watchdog timer. - */ -#define IGB_WATCHDOG (10 * hz) - -/* - * This parameter controls when the driver calls the routine to reclaim - * transmit descriptors. Cleaning earlier seems a win. - */ -#define IGB_TX_CLEANUP_THRESHOLD (adapter->num_tx_desc / 2) - -/* - * This parameter controls whether or not autonegotation is enabled. - * 0 - Disable autonegotiation - * 1 - Enable autonegotiation - */ -#define DO_AUTO_NEG 1 - -/* - * This parameter control whether or not the driver will wait for - * autonegotiation to complete. - * 1 - Wait for autonegotiation to complete - * 0 - Don't wait for autonegotiation to complete - */ -#define WAIT_FOR_AUTO_NEG_DEFAULT 0 - -/* Tunables -- End */ - -#define AUTONEG_ADV_DEFAULT (ADVERTISE_10_HALF | ADVERTISE_10_FULL | \ - ADVERTISE_100_HALF | ADVERTISE_100_FULL | \ - ADVERTISE_1000_FULL) - -#define AUTO_ALL_MODES 0 - -/* PHY master/slave setting */ -#define IGB_MASTER_SLAVE e1000_ms_hw_default - -/* Support AutoMediaDetect for Marvell M88 PHY in i354 */ -#define IGB_MEDIA_RESET (1 << 0) - -/* - * Micellaneous constants - */ -#define IGB_INTEL_VENDOR_ID 0x8086 - -#define IGB_JUMBO_PBA 0x00000028 -#define IGB_DEFAULT_PBA 0x00000030 -#define IGB_SMARTSPEED_DOWNSHIFT 3 -#define IGB_SMARTSPEED_MAX 15 -#define IGB_MAX_LOOP 10 - -#define IGB_RX_PTHRESH ((hw->mac.type == e1000_i354) ? 12 : \ - ((hw->mac.type <= e1000_82576) ? 16 : 8)) -#define IGB_RX_HTHRESH 8 -#define IGB_RX_WTHRESH ((hw->mac.type == e1000_82576 && \ - adapter->msix_mem) ? 1 : 4) - -#define IGB_TX_PTHRESH ((hw->mac.type == e1000_i354) ? 20 : 8) -#define IGB_TX_HTHRESH 1 -#define IGB_TX_WTHRESH ((hw->mac.type != e1000_82575 && \ - adapter->msix_mem) ? 1 : 16) - -#define MAX_NUM_MULTICAST_ADDRESSES 128 -#define PCI_ANY_ID (~0U) -#define ETHER_ALIGN 2 -#define IGB_TX_BUFFER_SIZE ((uint32_t) 1514) -#define IGB_FC_PAUSE_TIME 0x0680 -#define IGB_EEPROM_APME 0x400; -/* Queue minimum free for use */ -#define IGB_QUEUE_THRESHOLD (adapter->num_tx_desc / 8) - -/* - * TDBA/RDBA should be aligned on 16 byte boundary. But TDLEN/RDLEN should be - * multiple of 128 bytes. So we align TDBA/RDBA on 128 byte boundary. This will - * also optimize cache line size effect. H/W supports up to cache line size 128. - */ -#define IGB_DBA_ALIGN 128 - -#define SPEED_MODE_BIT (1<<21) /* On PCI-E MACs only */ - -/* PCI Config defines */ -#define IGB_MSIX_BAR 3 - -/* Defines for printing debug information */ -#define DEBUG_INIT 0 -#define DEBUG_IOCTL 0 -#define DEBUG_HW 0 - -#define INIT_DEBUGOUT(S) if (DEBUG_INIT) printf(S "\n") -#define INIT_DEBUGOUT1(S, A) if (DEBUG_INIT) printf(S "\n", A) -#define INIT_DEBUGOUT2(S, A, B) if (DEBUG_INIT) printf(S "\n", A, B) -#define IOCTL_DEBUGOUT(S) if (DEBUG_IOCTL) printf(S "\n") -#define IOCTL_DEBUGOUT1(S, A) if (DEBUG_IOCTL) printf(S "\n", A) -#define IOCTL_DEBUGOUT2(S, A, B) if (DEBUG_IOCTL) printf(S "\n", A, B) -#define HW_DEBUGOUT(S) if (DEBUG_HW) printf(S "\n") -#define HW_DEBUGOUT1(S, A) if (DEBUG_HW) printf(S "\n", A) -#define HW_DEBUGOUT2(S, A, B) if (DEBUG_HW) printf(S "\n", A, B) - -#define IGB_MAX_SCATTER 40 -#define IGB_VFTA_SIZE 128 -#define IGB_BR_SIZE 4096 /* ring buf size */ -#define IGB_TSO_SIZE (65535 + sizeof(struct ether_vlan_header)) -#define IGB_TSO_SEG_SIZE 4096 /* Max dma segment size */ -#define IGB_TXPBSIZE 20408 -#define IGB_HDR_BUF 128 -#define IGB_PKTTYPE_MASK 0x0000FFF0 -#define IGB_DMCTLX_DCFLUSH_DIS 0x80000000 /* Disable DMA Coalesce Flush */ -#define ETH_ZLEN 60 -#define ETH_ADDR_LEN 6 - -/* Offload bits in mbuf flag */ -#if __FreeBSD_version >= 1000000 -#define CSUM_OFFLOAD_IPV4 (CSUM_IP|CSUM_IP_TCP|CSUM_IP_UDP|CSUM_IP_SCTP) -#define CSUM_OFFLOAD_IPV6 (CSUM_IP6_TCP|CSUM_IP6_UDP|CSUM_IP6_SCTP) -#define CSUM_OFFLOAD (CSUM_OFFLOAD_IPV4|CSUM_OFFLOAD_IPV6) -#elif __FreeBSD_version >= 800000 -#define CSUM_OFFLOAD (CSUM_IP|CSUM_TCP|CSUM_UDP|CSUM_SCTP) -#else -#define CSUM_OFFLOAD (CSUM_IP|CSUM_TCP|CSUM_UDP) -#endif - -/* Define the starting Interrupt rate per Queue */ -#define IGB_INTS_PER_SEC 8000 -#define IGB_DEFAULT_ITR ((1000000/IGB_INTS_PER_SEC) << 2) - -#define IGB_LINK_ITR 2000 -#define I210_LINK_DELAY 1000 - -/* Precision Time Sync (IEEE 1588) defines */ -#define ETHERTYPE_IEEE1588 0x88F7 -#define PICOSECS_PER_TICK 20833 -#define TSYNC_PORT 319 /* UDP port for the protocol */ - -/* - * Bus dma allocation structure used by - * e1000_dma_malloc and e1000_dma_free. - */ -struct igb_dma_alloc { - bus_addr_t dma_paddr; - caddr_t dma_vaddr; - bus_dma_tag_t dma_tag; - bus_dmamap_t dma_map; - bus_dma_segment_t dma_seg; - int dma_nseg; -}; - - -/* -** Driver queue struct: this is the interrupt container -** for the associated tx and rx ring. -*/ -struct igb_queue { - struct adapter *adapter; - u32 msix; /* This queue's MSIX vector */ - u32 eims; /* This queue's EIMS bit */ - u32 eitr_setting; - struct resource *res; - void *tag; - struct tx_ring *txr; - struct rx_ring *rxr; - struct task que_task; - struct taskqueue *tq; - u64 irqs; -}; - -/* - * The transmit ring, one per queue - */ -struct tx_ring { - struct adapter *adapter; - struct mtx tx_mtx; - u32 me; - int watchdog_time; - union e1000_adv_tx_desc *tx_base; - struct igb_tx_buf *tx_buffers; - struct igb_dma_alloc txdma; - volatile u16 tx_avail; - u16 next_avail_desc; - u16 next_to_clean; - u16 num_desc; - enum { - IGB_QUEUE_IDLE = 1, - IGB_QUEUE_WORKING = 2, - IGB_QUEUE_HUNG = 4, - IGB_QUEUE_DEPLETED = 8, - } queue_status; - u32 txd_cmd; - bus_dma_tag_t txtag; - char mtx_name[16]; -#ifndef IGB_LEGACY_TX - struct buf_ring *br; - struct task txq_task; -#endif - u32 bytes; /* used for AIM */ - u32 packets; - /* Soft Stats */ - unsigned long tso_tx; - unsigned long no_tx_map_avail; - unsigned long no_tx_dma_setup; - u64 no_desc_avail; - u64 total_packets; -}; - -/* - * Receive ring: one per queue - */ -struct rx_ring { - struct adapter *adapter; - u32 me; - struct igb_dma_alloc rxdma; - union e1000_adv_rx_desc *rx_base; - struct lro_ctrl lro; - bool lro_enabled; - bool hdr_split; - struct mtx rx_mtx; - char mtx_name[16]; - u32 next_to_refresh; - u32 next_to_check; - struct igb_rx_buf *rx_buffers; - bus_dma_tag_t htag; /* dma tag for rx head */ - bus_dma_tag_t ptag; /* dma tag for rx packet */ - /* - * First/last mbuf pointers, for - * collecting multisegment RX packets. - */ - struct mbuf *fmp; - struct mbuf *lmp; - - u32 bytes; - u32 packets; - int rdt; - int rdh; - - /* Soft stats */ - u64 rx_split_packets; - u64 rx_discarded; - u64 rx_packets; - u64 rx_bytes; -}; - -struct adapter { - struct ifnet *ifp; - struct e1000_hw hw; - - struct e1000_osdep osdep; - device_t dev; - struct cdev *led_dev; - - struct resource *pci_mem; - struct resource *msix_mem; - int memrid; - - /* - * Interrupt resources: this set is - * either used for legacy, or for Link - * when doing MSIX - */ - void *tag; - struct resource *res; - - struct ifmedia media; - struct callout timer; - int msix; - int if_flags; - int pause_frames; - - struct mtx core_mtx; - - eventhandler_tag vlan_attach; - eventhandler_tag vlan_detach; - - u16 num_vlans; - u16 num_queues; - - /* - ** Shadow VFTA table, this is needed because - ** the real vlan filter table gets cleared during - ** a soft reset and the driver needs to be able - ** to repopulate it. - */ - u32 shadow_vfta[IGB_VFTA_SIZE]; - - /* Info about the interface */ - u32 optics; - u32 fc; /* local flow ctrl setting */ - int advertise; /* link speeds */ - bool link_active; - u16 max_frame_size; - u16 num_segs; - u16 link_speed; - bool link_up; - u32 linkvec; - u16 link_duplex; - u32 dmac; - int link_mask; - - /* Flags */ - u32 flags; - - /* Mbuf cluster size */ - u32 rx_mbuf_sz; - - /* Support for pluggable optics */ - bool sfp_probe; - struct task link_task; /* Link tasklet */ - struct task mod_task; /* SFP tasklet */ - struct task msf_task; /* Multispeed Fiber */ - struct taskqueue *tq; - - /* - ** Queues: - ** This is the irq holder, it has - ** and RX/TX pair or rings associated - ** with it. - */ - struct igb_queue *queues; - - /* - * Transmit rings: - * Allocated at run time, an array of rings. - */ - struct tx_ring *tx_rings; - u32 num_tx_desc; - - /* - * Receive rings: - * Allocated at run time, an array of rings. - */ - struct rx_ring *rx_rings; - u64 que_mask; - u32 num_rx_desc; - - /* Multicast array memory */ - u8 *mta; - - /* Misc stats maintained by the driver */ - unsigned long device_control; - unsigned long dropped_pkts; - unsigned long eint_mask; - unsigned long int_mask; - unsigned long link_irq; - unsigned long mbuf_defrag_failed; - unsigned long no_tx_dma_setup; - unsigned long packet_buf_alloc_rx; - unsigned long packet_buf_alloc_tx; - unsigned long rx_control; - unsigned long rx_overruns; - unsigned long watchdog_events; - - /* Used in pf and vf */ - void *stats; - - int enable_aim; - int has_manage; - int wol; - int rx_process_limit; - int tx_process_limit; - u16 vf_ifp; /* a VF interface */ - bool in_detach; /* Used only in igb_ioctl */ - -}; - -/* ****************************************************************************** - * vendor_info_array - * - * This array contains the list of Subvendor/Subdevice IDs on which the driver - * should load. - * - * ******************************************************************************/ -typedef struct _igb_vendor_info_t { - unsigned int vendor_id; - unsigned int device_id; - unsigned int subvendor_id; - unsigned int subdevice_id; - unsigned int index; -} igb_vendor_info_t; - -struct igb_tx_buf { - union e1000_adv_tx_desc *eop; - struct mbuf *m_head; - bus_dmamap_t map; -}; - -struct igb_rx_buf { - struct mbuf *m_head; - struct mbuf *m_pack; - bus_dmamap_t hmap; /* bus_dma map for header */ - bus_dmamap_t pmap; /* bus_dma map for packet */ -}; - -/* -** Find the number of unrefreshed RX descriptors -*/ -static inline u16 -igb_rx_unrefreshed(struct rx_ring *rxr) -{ - struct adapter *adapter = rxr->adapter; - - if (rxr->next_to_check > rxr->next_to_refresh) - return (rxr->next_to_check - rxr->next_to_refresh - 1); - else - return ((adapter->num_rx_desc + rxr->next_to_check) - - rxr->next_to_refresh - 1); -} - -#define IGB_CORE_LOCK_INIT(_sc, _name) \ - mtx_init(&(_sc)->core_mtx, _name, "IGB Core Lock", MTX_DEF) -#define IGB_CORE_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->core_mtx) -#define IGB_CORE_LOCK(_sc) mtx_lock(&(_sc)->core_mtx) -#define IGB_CORE_UNLOCK(_sc) mtx_unlock(&(_sc)->core_mtx) -#define IGB_CORE_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->core_mtx, MA_OWNED) - -#define IGB_TX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->tx_mtx) -#define IGB_TX_LOCK(_sc) mtx_lock(&(_sc)->tx_mtx) -#define IGB_TX_UNLOCK(_sc) mtx_unlock(&(_sc)->tx_mtx) -#define IGB_TX_TRYLOCK(_sc) mtx_trylock(&(_sc)->tx_mtx) -#define IGB_TX_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->tx_mtx, MA_OWNED) - -#define IGB_RX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->rx_mtx) -#define IGB_RX_LOCK(_sc) mtx_lock(&(_sc)->rx_mtx) -#define IGB_RX_UNLOCK(_sc) mtx_unlock(&(_sc)->rx_mtx) -#define IGB_RX_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->rx_mtx, MA_OWNED) - -#define UPDATE_VF_REG(reg, last, cur) \ -{ \ - u32 new = E1000_READ_REG(hw, reg); \ - if (new < last) \ - cur += 0x100000000LL; \ - last = new; \ - cur &= 0xFFFFFFFF00000000LL; \ - cur |= new; \ -} - -#if __FreeBSD_version >= 800000 && __FreeBSD_version < 800504 -static __inline int -drbr_needs_enqueue(struct ifnet *ifp, struct buf_ring *br) -{ -#ifdef ALTQ - if (ALTQ_IS_ENABLED(&ifp->if_snd)) - return (1); -#endif - return (!buf_ring_empty(br)); -} -#endif - -#endif /* _IF_IGB_H_ */ - - diff --git a/freebsd/sys/dev/e1000/if_lem.c b/freebsd/sys/dev/e1000/if_lem.c deleted file mode 100644 index b3da1bdd..00000000 --- a/freebsd/sys/dev/e1000/if_lem.c +++ /dev/null @@ -1,4732 +0,0 @@ -#include <machine/rtems-bsd-kernel-space.h> - -/****************************************************************************** - - Copyright (c) 2001-2015, Intel Corporation - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - -******************************************************************************/ -/*$FreeBSD$*/ - -/* - * Uncomment the following extensions for better performance in a VM, - * especially if you have support in the hypervisor. - * See http://info.iet.unipi.it/~luigi/netmap/ - */ -// #define BATCH_DISPATCH -// #define NIC_SEND_COMBINING - -#include <rtems/bsd/local/opt_inet.h> -#include <rtems/bsd/local/opt_inet6.h> - -#ifdef HAVE_KERNEL_OPTION_HEADERS -#include <rtems/bsd/local/opt_device_polling.h> -#endif - -#include <rtems/bsd/sys/param.h> -#include <sys/systm.h> -#include <sys/buf_ring.h> -#include <sys/bus.h> -#include <sys/endian.h> -#include <sys/kernel.h> -#include <sys/kthread.h> -#include <sys/malloc.h> -#include <sys/mbuf.h> -#include <sys/module.h> -#include <sys/rman.h> -#include <sys/socket.h> -#include <sys/sockio.h> -#include <sys/sysctl.h> -#include <sys/taskqueue.h> -#include <sys/eventhandler.h> -#include <machine/bus.h> -#include <machine/resource.h> - -#include <net/bpf.h> -#include <net/ethernet.h> -#include <net/if.h> -#include <net/if_var.h> -#include <net/if_arp.h> -#include <net/if_dl.h> -#include <net/if_media.h> - -#include <net/if_types.h> -#include <net/if_vlan_var.h> - -#include <netinet/in_systm.h> -#include <netinet/in.h> -#include <netinet/if_ether.h> -#include <netinet/ip.h> -#include <netinet/ip6.h> -#include <netinet/tcp.h> -#include <netinet/udp.h> - -#include <machine/in_cksum.h> -#include <dev/led/led.h> -#include <dev/pci/pcivar.h> -#include <dev/pci/pcireg.h> - -#include "e1000_api.h" -#include "if_lem.h" - -/********************************************************************* - * Legacy Em Driver version: - *********************************************************************/ -char lem_driver_version[] = "1.1.0"; - -/********************************************************************* - * PCI Device ID Table - * - * Used by probe to select devices to load on - * Last field stores an index into e1000_strings - * Last entry must be all 0s - * - * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } - *********************************************************************/ - -static em_vendor_info_t lem_vendor_info_array[] = -{ - /* Intel(R) PRO/1000 Network Connection */ - { 0x8086, E1000_DEV_ID_82540EM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82540EM_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82540EP, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82540EP_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82540EP_LP, PCI_ANY_ID, PCI_ANY_ID, 0}, - - { 0x8086, E1000_DEV_ID_82541EI, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82541ER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82541ER_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82541EI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82541GI, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82541GI_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82541GI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0}, - - { 0x8086, E1000_DEV_ID_82542, PCI_ANY_ID, PCI_ANY_ID, 0}, - - { 0x8086, E1000_DEV_ID_82543GC_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82543GC_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - - { 0x8086, E1000_DEV_ID_82544EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82544EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82544GC_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82544GC_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, - - { 0x8086, E1000_DEV_ID_82545EM_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82545EM_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82545GM_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82545GM_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82545GM_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, - - { 0x8086, E1000_DEV_ID_82546EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82546EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82546GB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82546GB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82546GB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82546GB_PCIE, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3, - PCI_ANY_ID, PCI_ANY_ID, 0}, - - { 0x8086, E1000_DEV_ID_82547EI, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82547EI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82547GI, PCI_ANY_ID, PCI_ANY_ID, 0}, - /* required last entry */ - { 0, 0, 0, 0, 0} -}; - -/********************************************************************* - * Table of branding strings for all supported NICs. - *********************************************************************/ - -static char *lem_strings[] = { - "Intel(R) PRO/1000 Legacy Network Connection" -}; - -/********************************************************************* - * Function prototypes - *********************************************************************/ -static int lem_probe(device_t); -static int lem_attach(device_t); -static int lem_detach(device_t); -static int lem_shutdown(device_t); -static int lem_suspend(device_t); -static int lem_resume(device_t); -static void lem_start(if_t); -static void lem_start_locked(if_t ifp); -static int lem_ioctl(if_t, u_long, caddr_t); -static uint64_t lem_get_counter(if_t, ift_counter); -static void lem_init(void *); -static void lem_init_locked(struct adapter *); -static void lem_stop(void *); -static void lem_media_status(if_t, struct ifmediareq *); -static int lem_media_change(if_t); -static void lem_identify_hardware(struct adapter *); -static int lem_allocate_pci_resources(struct adapter *); -static int lem_allocate_irq(struct adapter *adapter); -static void lem_free_pci_resources(struct adapter *); -static void lem_local_timer(void *); -static int lem_hardware_init(struct adapter *); -static int lem_setup_interface(device_t, struct adapter *); -static void lem_setup_transmit_structures(struct adapter *); -static void lem_initialize_transmit_unit(struct adapter *); -static int lem_setup_receive_structures(struct adapter *); -static void lem_initialize_receive_unit(struct adapter *); -static void lem_enable_intr(struct adapter *); -static void lem_disable_intr(struct adapter *); -static void lem_free_transmit_structures(struct adapter *); -static void lem_free_receive_structures(struct adapter *); -static void lem_update_stats_counters(struct adapter *); -static void lem_add_hw_stats(struct adapter *adapter); -static void lem_txeof(struct adapter *); -static void lem_tx_purge(struct adapter *); -static int lem_allocate_receive_structures(struct adapter *); -static int lem_allocate_transmit_structures(struct adapter *); -static bool lem_rxeof(struct adapter *, int, int *); -#ifndef __NO_STRICT_ALIGNMENT -static int lem_fixup_rx(struct adapter *); -#endif -static void lem_receive_checksum(struct adapter *, struct e1000_rx_desc *, - struct mbuf *); -static void lem_transmit_checksum_setup(struct adapter *, struct mbuf *, - u32 *, u32 *); -static void lem_set_promisc(struct adapter *); -static void lem_disable_promisc(struct adapter *); -static void lem_set_multi(struct adapter *); -static void lem_update_link_status(struct adapter *); -static int lem_get_buf(struct adapter *, int); -static void lem_register_vlan(void *, if_t, u16); -static void lem_unregister_vlan(void *, if_t, u16); -static void lem_setup_vlan_hw_support(struct adapter *); -static int lem_xmit(struct adapter *, struct mbuf **); -static void lem_smartspeed(struct adapter *); -static int lem_82547_fifo_workaround(struct adapter *, int); -static void lem_82547_update_fifo_head(struct adapter *, int); -static int lem_82547_tx_fifo_reset(struct adapter *); -static void lem_82547_move_tail(void *); -static int lem_dma_malloc(struct adapter *, bus_size_t, - struct em_dma_alloc *, int); -static void lem_dma_free(struct adapter *, struct em_dma_alloc *); -static int lem_sysctl_nvm_info(SYSCTL_HANDLER_ARGS); -static void lem_print_nvm_info(struct adapter *); -static int lem_is_valid_ether_addr(u8 *); -static u32 lem_fill_descriptors (bus_addr_t address, u32 length, - PDESC_ARRAY desc_array); -static int lem_sysctl_int_delay(SYSCTL_HANDLER_ARGS); -static void lem_add_int_delay_sysctl(struct adapter *, const char *, - const char *, struct em_int_delay_info *, int, int); -static void lem_set_flow_cntrl(struct adapter *, const char *, - const char *, int *, int); -/* Management and WOL Support */ -static void lem_init_manageability(struct adapter *); -static void lem_release_manageability(struct adapter *); -static void lem_get_hw_control(struct adapter *); -static void lem_release_hw_control(struct adapter *); -static void lem_get_wakeup(device_t); -static void lem_enable_wakeup(device_t); -static int lem_enable_phy_wakeup(struct adapter *); -static void lem_led_func(void *, int); - -static void lem_intr(void *); -static int lem_irq_fast(void *); -static void lem_handle_rxtx(void *context, int pending); -static void lem_handle_link(void *context, int pending); -static void lem_add_rx_process_limit(struct adapter *, const char *, - const char *, int *, int); - -#ifdef DEVICE_POLLING -static poll_handler_t lem_poll; -#endif /* POLLING */ - -/********************************************************************* - * FreeBSD Device Interface Entry Points - *********************************************************************/ - -static device_method_t lem_methods[] = { - /* Device interface */ - DEVMETHOD(device_probe, lem_probe), - DEVMETHOD(device_attach, lem_attach), - DEVMETHOD(device_detach, lem_detach), - DEVMETHOD(device_shutdown, lem_shutdown), - DEVMETHOD(device_suspend, lem_suspend), - DEVMETHOD(device_resume, lem_resume), - DEVMETHOD_END -}; - -static driver_t lem_driver = { - "em", lem_methods, sizeof(struct adapter), -}; - -extern devclass_t em_devclass; -DRIVER_MODULE(lem, pci, lem_driver, em_devclass, 0, 0); -MODULE_DEPEND(lem, pci, 1, 1, 1); -MODULE_DEPEND(lem, ether, 1, 1, 1); -#ifdef DEV_NETMAP -MODULE_DEPEND(lem, netmap, 1, 1, 1); -#endif /* DEV_NETMAP */ - -/********************************************************************* - * Tunable default values. - *********************************************************************/ - -#define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000) -#define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024) - -#define MAX_INTS_PER_SEC 8000 -#define DEFAULT_ITR (1000000000/(MAX_INTS_PER_SEC * 256)) - -static int lem_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV); -static int lem_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR); -static int lem_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV); -static int lem_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV); -/* - * increase lem_rxd and lem_txd to at least 2048 in netmap mode - * for better performance. - */ -static int lem_rxd = EM_DEFAULT_RXD; -static int lem_txd = EM_DEFAULT_TXD; -static int lem_smart_pwr_down = FALSE; - -/* Controls whether promiscuous also shows bad packets */ -static int lem_debug_sbp = FALSE; - -TUNABLE_INT("hw.em.tx_int_delay", &lem_tx_int_delay_dflt); -TUNABLE_INT("hw.em.rx_int_delay", &lem_rx_int_delay_dflt); -TUNABLE_INT("hw.em.tx_abs_int_delay", &lem_tx_abs_int_delay_dflt); -TUNABLE_INT("hw.em.rx_abs_int_delay", &lem_rx_abs_int_delay_dflt); -TUNABLE_INT("hw.em.rxd", &lem_rxd); -TUNABLE_INT("hw.em.txd", &lem_txd); -TUNABLE_INT("hw.em.smart_pwr_down", &lem_smart_pwr_down); -TUNABLE_INT("hw.em.sbp", &lem_debug_sbp); - -/* Interrupt style - default to fast */ -static int lem_use_legacy_irq = 0; -TUNABLE_INT("hw.em.use_legacy_irq", &lem_use_legacy_irq); - -/* How many packets rxeof tries to clean at a time */ -static int lem_rx_process_limit = 100; -TUNABLE_INT("hw.em.rx_process_limit", &lem_rx_process_limit); - -/* Flow control setting - default to FULL */ -static int lem_fc_setting = e1000_fc_full; -TUNABLE_INT("hw.em.fc_setting", &lem_fc_setting); - -/* Global used in WOL setup with multiport cards */ -static int global_quad_port_a = 0; - -#ifdef DEV_NETMAP /* see ixgbe.c for details */ -#include <dev/netmap/if_lem_netmap.h> -#endif /* DEV_NETMAP */ - -/********************************************************************* - * Device identification routine - * - * em_probe determines if the driver should be loaded on - * adapter based on PCI vendor/device id of the adapter. - * - * return BUS_PROBE_DEFAULT on success, positive on failure - *********************************************************************/ - -static int -lem_probe(device_t dev) -{ - char adapter_name[60]; - u16 pci_vendor_id = 0; - u16 pci_device_id = 0; - u16 pci_subvendor_id = 0; - u16 pci_subdevice_id = 0; - em_vendor_info_t *ent; - - INIT_DEBUGOUT("em_probe: begin"); - - pci_vendor_id = pci_get_vendor(dev); - if (pci_vendor_id != EM_VENDOR_ID) - return (ENXIO); - - pci_device_id = pci_get_device(dev); - pci_subvendor_id = pci_get_subvendor(dev); - pci_subdevice_id = pci_get_subdevice(dev); - - ent = lem_vendor_info_array; - while (ent->vendor_id != 0) { - if ((pci_vendor_id == ent->vendor_id) && - (pci_device_id == ent->device_id) && - - ((pci_subvendor_id == ent->subvendor_id) || - (ent->subvendor_id == PCI_ANY_ID)) && - - ((pci_subdevice_id == ent->subdevice_id) || - (ent->subdevice_id == PCI_ANY_ID))) { - sprintf(adapter_name, "%s %s", - lem_strings[ent->index], - lem_driver_version); - device_set_desc_copy(dev, adapter_name); - return (BUS_PROBE_DEFAULT); - } - ent++; - } - - return (ENXIO); -} - -/********************************************************************* - * Device initialization routine - * - * The attach entry point is called when the driver is being loaded. - * This routine identifies the type of hardware, allocates all resources - * and initializes the hardware. - * - * return 0 on success, positive on failure - *********************************************************************/ - -static int -lem_attach(device_t dev) -{ - struct adapter *adapter; - int tsize, rsize; - int error = 0; - - INIT_DEBUGOUT("lem_attach: begin"); - - adapter = device_get_softc(dev); - adapter->dev = adapter->osdep.dev = dev; - EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); - EM_TX_LOCK_INIT(adapter, device_get_nameunit(dev)); - EM_RX_LOCK_INIT(adapter, device_get_nameunit(dev)); - - /* SYSCTL stuff */ - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, - lem_sysctl_nvm_info, "I", "NVM Information"); - - callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); - callout_init_mtx(&adapter->tx_fifo_timer, &adapter->tx_mtx, 0); - - /* Determine hardware and mac info */ - lem_identify_hardware(adapter); - - /* Setup PCI resources */ - if (lem_allocate_pci_resources(adapter)) { - device_printf(dev, "Allocation of PCI resources failed\n"); - error = ENXIO; - goto err_pci; - } - - /* Do Shared Code initialization */ - if (e1000_setup_init_funcs(&adapter->hw, TRUE)) { - device_printf(dev, "Setup of Shared code failed\n"); - error = ENXIO; - goto err_pci; - } - - e1000_get_bus_info(&adapter->hw); - - /* Set up some sysctls for the tunable interrupt delays */ - lem_add_int_delay_sysctl(adapter, "rx_int_delay", - "receive interrupt delay in usecs", &adapter->rx_int_delay, - E1000_REGISTER(&adapter->hw, E1000_RDTR), lem_rx_int_delay_dflt); - lem_add_int_delay_sysctl(adapter, "tx_int_delay", - "transmit interrupt delay in usecs", &adapter->tx_int_delay, - E1000_REGISTER(&adapter->hw, E1000_TIDV), lem_tx_int_delay_dflt); - if (adapter->hw.mac.type >= e1000_82540) { - lem_add_int_delay_sysctl(adapter, "rx_abs_int_delay", - "receive interrupt delay limit in usecs", - &adapter->rx_abs_int_delay, - E1000_REGISTER(&adapter->hw, E1000_RADV), - lem_rx_abs_int_delay_dflt); - lem_add_int_delay_sysctl(adapter, "tx_abs_int_delay", - "transmit interrupt delay limit in usecs", - &adapter->tx_abs_int_delay, - E1000_REGISTER(&adapter->hw, E1000_TADV), - lem_tx_abs_int_delay_dflt); - lem_add_int_delay_sysctl(adapter, "itr", - "interrupt delay limit in usecs/4", - &adapter->tx_itr, - E1000_REGISTER(&adapter->hw, E1000_ITR), - DEFAULT_ITR); - } - - /* Sysctls for limiting the amount of work done in the taskqueue */ - lem_add_rx_process_limit(adapter, "rx_processing_limit", - "max number of rx packets to process", &adapter->rx_process_limit, - lem_rx_process_limit); - -#ifdef NIC_SEND_COMBINING - /* Sysctls to control mitigation */ - lem_add_rx_process_limit(adapter, "sc_enable", - "driver TDT mitigation", &adapter->sc_enable, 0); -#endif /* NIC_SEND_COMBINING */ -#ifdef BATCH_DISPATCH - lem_add_rx_process_limit(adapter, "batch_enable", - "driver rx batch", &adapter->batch_enable, 0); -#endif /* BATCH_DISPATCH */ - - /* Sysctl for setting the interface flow control */ - lem_set_flow_cntrl(adapter, "flow_control", - "flow control setting", - &adapter->fc_setting, lem_fc_setting); - - /* - * Validate number of transmit and receive descriptors. It - * must not exceed hardware maximum, and must be multiple - * of E1000_DBA_ALIGN. - */ - if (((lem_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 || - (adapter->hw.mac.type >= e1000_82544 && lem_txd > EM_MAX_TXD) || - (adapter->hw.mac.type < e1000_82544 && lem_txd > EM_MAX_TXD_82543) || - (lem_txd < EM_MIN_TXD)) { - device_printf(dev, "Using %d TX descriptors instead of %d!\n", - EM_DEFAULT_TXD, lem_txd); - adapter->num_tx_desc = EM_DEFAULT_TXD; - } else - adapter->num_tx_desc = lem_txd; - if (((lem_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 || - (adapter->hw.mac.type >= e1000_82544 && lem_rxd > EM_MAX_RXD) || - (adapter->hw.mac.type < e1000_82544 && lem_rxd > EM_MAX_RXD_82543) || - (lem_rxd < EM_MIN_RXD)) { - device_printf(dev, "Using %d RX descriptors instead of %d!\n", - EM_DEFAULT_RXD, lem_rxd); - adapter->num_rx_desc = EM_DEFAULT_RXD; - } else - adapter->num_rx_desc = lem_rxd; - - adapter->hw.mac.autoneg = DO_AUTO_NEG; - adapter->hw.phy.autoneg_wait_to_complete = FALSE; - adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; - adapter->rx_buffer_len = 2048; - - e1000_init_script_state_82541(&adapter->hw, TRUE); - e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE); - - /* Copper options */ - if (adapter->hw.phy.media_type == e1000_media_type_copper) { - adapter->hw.phy.mdix = AUTO_ALL_MODES; - adapter->hw.phy.disable_polarity_correction = FALSE; - adapter->hw.phy.ms_type = EM_MASTER_SLAVE; - } - - /* - * Set the frame limits assuming - * standard ethernet sized frames. - */ - adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; - adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE; - - /* - * This controls when hardware reports transmit completion - * status. - */ - adapter->hw.mac.report_tx_early = 1; - - /* - * It seems that the descriptor DMA engine on some PCI cards - * fetches memory past the end of the last descriptor in the - * ring. These reads are problematic when VT-d (DMAR) busdma - * is used. Allocate the scratch space to avoid getting - * faults from DMAR, by requesting scratch memory for one more - * descriptor. - */ - tsize = roundup2((adapter->num_tx_desc + 1) * - sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); - - /* Allocate Transmit Descriptor ring */ - if (lem_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_NOWAIT)) { - device_printf(dev, "Unable to allocate tx_desc memory\n"); - error = ENOMEM; - goto err_tx_desc; - } - adapter->tx_desc_base = - (struct e1000_tx_desc *)adapter->txdma.dma_vaddr; - - /* - * See comment above txdma allocation for rationale behind +1. - */ - rsize = roundup2((adapter->num_rx_desc + 1) * - sizeof(struct e1000_rx_desc), EM_DBA_ALIGN); - - /* Allocate Receive Descriptor ring */ - if (lem_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) { - device_printf(dev, "Unable to allocate rx_desc memory\n"); - error = ENOMEM; - goto err_rx_desc; - } - adapter->rx_desc_base = - (struct e1000_rx_desc *)adapter->rxdma.dma_vaddr; - - /* Allocate multicast array memory. */ - adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN * - MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); - if (adapter->mta == NULL) { - device_printf(dev, "Can not allocate multicast setup array\n"); - error = ENOMEM; - goto err_hw_init; - } - - /* - ** Start from a known state, this is - ** important in reading the nvm and - ** mac from that. - */ - e1000_reset_hw(&adapter->hw); - - /* Make sure we have a good EEPROM before we read from it */ - if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { - /* - ** Some PCI-E parts fail the first check due to - ** the link being in sleep state, call it again, - ** if it fails a second time its a real issue. - */ - if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { - device_printf(dev, - "The EEPROM Checksum Is Not Valid\n"); - error = EIO; - goto err_hw_init; - } - } - - /* Copy the permanent MAC address out of the EEPROM */ - if (e1000_read_mac_addr(&adapter->hw) < 0) { - device_printf(dev, "EEPROM read error while reading MAC" - " address\n"); - error = EIO; - goto err_hw_init; - } - - if (!lem_is_valid_ether_addr(adapter->hw.mac.addr)) { - device_printf(dev, "Invalid MAC address\n"); - error = EIO; - goto err_hw_init; - } - - /* Initialize the hardware */ - if (lem_hardware_init(adapter)) { - device_printf(dev, "Unable to initialize the hardware\n"); - error = EIO; - goto err_hw_init; - } - - /* Allocate transmit descriptors and buffers */ - if (lem_allocate_transmit_structures(adapter)) { - device_printf(dev, "Could not setup transmit structures\n"); - error = ENOMEM; - goto err_tx_struct; - } - - /* Allocate receive descriptors and buffers */ - if (lem_allocate_receive_structures(adapter)) { - device_printf(dev, "Could not setup receive structures\n"); - error = ENOMEM; - goto err_rx_struct; - } - - /* - ** Do interrupt configuration - */ - error = lem_allocate_irq(adapter); - if (error) - goto err_rx_struct; - - /* - * Get Wake-on-Lan and Management info for later use - */ - lem_get_wakeup(dev); - - /* Setup OS specific network interface */ - if (lem_setup_interface(dev, adapter) != 0) - goto err_rx_struct; - - /* Initialize statistics */ - lem_update_stats_counters(adapter); - - adapter->hw.mac.get_link_status = 1; - lem_update_link_status(adapter); - - /* Indicate SOL/IDER usage */ - if (e1000_check_reset_block(&adapter->hw)) - device_printf(dev, - "PHY reset is blocked due to SOL/IDER session.\n"); - - /* Do we need workaround for 82544 PCI-X adapter? */ - if (adapter->hw.bus.type == e1000_bus_type_pcix && - adapter->hw.mac.type == e1000_82544) - adapter->pcix_82544 = TRUE; - else - adapter->pcix_82544 = FALSE; - - /* Register for VLAN events */ - adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, - lem_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); - adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, - lem_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); - - lem_add_hw_stats(adapter); - - /* Non-AMT based hardware can now take control from firmware */ - if (adapter->has_manage && !adapter->has_amt) - lem_get_hw_control(adapter); - - /* Tell the stack that the interface is not active */ - if_setdrvflagbits(adapter->ifp, 0, IFF_DRV_OACTIVE | IFF_DRV_RUNNING); - - adapter->led_dev = led_create(lem_led_func, adapter, - device_get_nameunit(dev)); - -#ifdef DEV_NETMAP - lem_netmap_attach(adapter); -#endif /* DEV_NETMAP */ - INIT_DEBUGOUT("lem_attach: end"); - - return (0); - -err_rx_struct: - lem_free_transmit_structures(adapter); -err_tx_struct: -err_hw_init: - lem_release_hw_control(adapter); - lem_dma_free(adapter, &adapter->rxdma); -err_rx_desc: - lem_dma_free(adapter, &adapter->txdma); -err_tx_desc: -err_pci: - if (adapter->ifp != (void *)NULL) - if_free(adapter->ifp); - lem_free_pci_resources(adapter); - free(adapter->mta, M_DEVBUF); - EM_TX_LOCK_DESTROY(adapter); - EM_RX_LOCK_DESTROY(adapter); - EM_CORE_LOCK_DESTROY(adapter); - - return (error); -} - -/********************************************************************* - * Device removal routine - * - * The detach entry point is called when the driver is being removed. - * This routine stops the adapter and deallocates all the resources - * that were allocated for driver operation. - * - * return 0 on success, positive on failure - *********************************************************************/ - -static int -lem_detach(device_t dev) -{ - struct adapter *adapter = device_get_softc(dev); - if_t ifp = adapter->ifp; - - INIT_DEBUGOUT("em_detach: begin"); - - /* Make sure VLANS are not using driver */ - if (if_vlantrunkinuse(ifp)) { - device_printf(dev,"Vlan in use, detach first\n"); - return (EBUSY); - } - -#ifdef DEVICE_POLLING - if (if_getcapenable(ifp) & IFCAP_POLLING) - ether_poll_deregister(ifp); -#endif - - if (adapter->led_dev != NULL) - led_destroy(adapter->led_dev); - - EM_CORE_LOCK(adapter); - EM_TX_LOCK(adapter); - adapter->in_detach = 1; - lem_stop(adapter); - e1000_phy_hw_reset(&adapter->hw); - - lem_release_manageability(adapter); - - EM_TX_UNLOCK(adapter); - EM_CORE_UNLOCK(adapter); - - /* Unregister VLAN events */ - if (adapter->vlan_attach != NULL) - EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); - if (adapter->vlan_detach != NULL) - EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); - - ether_ifdetach(adapter->ifp); - callout_drain(&adapter->timer); - callout_drain(&adapter->tx_fifo_timer); - -#ifdef DEV_NETMAP - netmap_detach(ifp); -#endif /* DEV_NETMAP */ - lem_free_pci_resources(adapter); - bus_generic_detach(dev); - if_free(ifp); - - lem_free_transmit_structures(adapter); - lem_free_receive_structures(adapter); - - /* Free Transmit Descriptor ring */ - if (adapter->tx_desc_base) { - lem_dma_free(adapter, &adapter->txdma); - adapter->tx_desc_base = NULL; - } - - /* Free Receive Descriptor ring */ - if (adapter->rx_desc_base) { - lem_dma_free(adapter, &adapter->rxdma); - adapter->rx_desc_base = NULL; - } - - lem_release_hw_control(adapter); - free(adapter->mta, M_DEVBUF); - EM_TX_LOCK_DESTROY(adapter); - EM_RX_LOCK_DESTROY(adapter); - EM_CORE_LOCK_DESTROY(adapter); - - return (0); -} - -/********************************************************************* - * - * Shutdown entry point - * - **********************************************************************/ - -static int -lem_shutdown(device_t dev) -{ - return lem_suspend(dev); -} - -/* - * Suspend/resume device methods. - */ -static int -lem_suspend(device_t dev) -{ - struct adapter *adapter = device_get_softc(dev); - - EM_CORE_LOCK(adapter); - - lem_release_manageability(adapter); - lem_release_hw_control(adapter); - lem_enable_wakeup(dev); - - EM_CORE_UNLOCK(adapter); - - return bus_generic_suspend(dev); -} - -static int -lem_resume(device_t dev) -{ - struct adapter *adapter = device_get_softc(dev); - if_t ifp = adapter->ifp; - - EM_CORE_LOCK(adapter); - lem_init_locked(adapter); - lem_init_manageability(adapter); - EM_CORE_UNLOCK(adapter); - lem_start(ifp); - - return bus_generic_resume(dev); -} - - -static void -lem_start_locked(if_t ifp) -{ - struct adapter *adapter = if_getsoftc(ifp); - struct mbuf *m_head; - - EM_TX_LOCK_ASSERT(adapter); - - if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != - IFF_DRV_RUNNING) - return; - if (!adapter->link_active) - return; - - /* - * Force a cleanup if number of TX descriptors - * available hits the threshold - */ - if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) { - lem_txeof(adapter); - /* Now do we at least have a minimal? */ - if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) { - adapter->no_tx_desc_avail1++; - return; - } - } - - while (!if_sendq_empty(ifp)) { - m_head = if_dequeue(ifp); - - if (m_head == NULL) - break; - /* - * Encapsulation can modify our pointer, and or make it - * NULL on failure. In that event, we can't requeue. - */ - if (lem_xmit(adapter, &m_head)) { - if (m_head == NULL) - break; - if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0); - if_sendq_prepend(ifp, m_head); - break; - } - - /* Send a copy of the frame to the BPF listener */ - if_etherbpfmtap(ifp, m_head); - - /* Set timeout in case hardware has problems transmitting. */ - adapter->watchdog_check = TRUE; - adapter->watchdog_time = ticks; - } - if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) - if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0); - - return; -} - -static void -lem_start(if_t ifp) -{ - struct adapter *adapter = if_getsoftc(ifp); - - EM_TX_LOCK(adapter); - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) - lem_start_locked(ifp); - EM_TX_UNLOCK(adapter); -} - -/********************************************************************* - * Ioctl entry point - * - * em_ioctl is called when the user wants to configure the - * interface. - * - * return 0 on success, positive on failure - **********************************************************************/ - -static int -lem_ioctl(if_t ifp, u_long command, caddr_t data) -{ - struct adapter *adapter = if_getsoftc(ifp); - struct ifreq *ifr = (struct ifreq *)data; -#if defined(INET) || defined(INET6) - struct ifaddr *ifa = (struct ifaddr *)data; -#endif - bool avoid_reset = FALSE; - int error = 0; - - if (adapter->in_detach) - return (error); - - switch (command) { - case SIOCSIFADDR: -#ifdef INET - if (ifa->ifa_addr->sa_family == AF_INET) - avoid_reset = TRUE; -#endif -#ifdef INET6 - if (ifa->ifa_addr->sa_family == AF_INET6) - avoid_reset = TRUE; -#endif - /* - ** Calling init results in link renegotiation, - ** so we avoid doing it when possible. - */ - if (avoid_reset) { - if_setflagbits(ifp, IFF_UP, 0); - if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) - lem_init(adapter); -#ifdef INET - if (!(if_getflags(ifp) & IFF_NOARP)) - arp_ifinit(ifp, ifa); -#endif - } else - error = ether_ioctl(ifp, command, data); - break; - case SIOCSIFMTU: - { - int max_frame_size; - - IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); - - EM_CORE_LOCK(adapter); - switch (adapter->hw.mac.type) { - case e1000_82542: - max_frame_size = ETHER_MAX_LEN; - break; - default: - max_frame_size = MAX_JUMBO_FRAME_SIZE; - } - if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - - ETHER_CRC_LEN) { - EM_CORE_UNLOCK(adapter); - error = EINVAL; - break; - } - - if_setmtu(ifp, ifr->ifr_mtu); - adapter->max_frame_size = - if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN; - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) - lem_init_locked(adapter); - EM_CORE_UNLOCK(adapter); - break; - } - case SIOCSIFFLAGS: - IOCTL_DEBUGOUT("ioctl rcv'd:\ - SIOCSIFFLAGS (Set Interface Flags)"); - EM_CORE_LOCK(adapter); - if (if_getflags(ifp) & IFF_UP) { - if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING)) { - if ((if_getflags(ifp) ^ adapter->if_flags) & - (IFF_PROMISC | IFF_ALLMULTI)) { - lem_disable_promisc(adapter); - lem_set_promisc(adapter); - } - } else - lem_init_locked(adapter); - } else - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { - EM_TX_LOCK(adapter); - lem_stop(adapter); - EM_TX_UNLOCK(adapter); - } - adapter->if_flags = if_getflags(ifp); - EM_CORE_UNLOCK(adapter); - break; - case SIOCADDMULTI: - case SIOCDELMULTI: - IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { - EM_CORE_LOCK(adapter); - lem_disable_intr(adapter); - lem_set_multi(adapter); - if (adapter->hw.mac.type == e1000_82542 && - adapter->hw.revision_id == E1000_REVISION_2) { - lem_initialize_receive_unit(adapter); - } -#ifdef DEVICE_POLLING - if (!(if_getcapenable(ifp) & IFCAP_POLLING)) -#endif - lem_enable_intr(adapter); - EM_CORE_UNLOCK(adapter); - } - break; - case SIOCSIFMEDIA: - /* Check SOL/IDER usage */ - EM_CORE_LOCK(adapter); - if (e1000_check_reset_block(&adapter->hw)) { - EM_CORE_UNLOCK(adapter); - device_printf(adapter->dev, "Media change is" - " blocked due to SOL/IDER session.\n"); - break; - } - EM_CORE_UNLOCK(adapter); - case SIOCGIFMEDIA: - IOCTL_DEBUGOUT("ioctl rcv'd: \ - SIOCxIFMEDIA (Get/Set Interface Media)"); - error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); - break; - case SIOCSIFCAP: - { - int mask, reinit; - - IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); - reinit = 0; - mask = ifr->ifr_reqcap ^ if_getcapenable(ifp); -#ifdef DEVICE_POLLING - if (mask & IFCAP_POLLING) { - if (ifr->ifr_reqcap & IFCAP_POLLING) { - error = ether_poll_register(lem_poll, ifp); - if (error) - return (error); - EM_CORE_LOCK(adapter); - lem_disable_intr(adapter); - if_setcapenablebit(ifp, IFCAP_POLLING, 0); - EM_CORE_UNLOCK(adapter); - } else { - error = ether_poll_deregister(ifp); - /* Enable interrupt even in error case */ - EM_CORE_LOCK(adapter); - lem_enable_intr(adapter); - if_setcapenablebit(ifp, 0, IFCAP_POLLING); - EM_CORE_UNLOCK(adapter); - } - } -#endif - if (mask & IFCAP_HWCSUM) { - if_togglecapenable(ifp, IFCAP_HWCSUM); - reinit = 1; - } - if (mask & IFCAP_VLAN_HWTAGGING) { - if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING); - reinit = 1; - } - if ((mask & IFCAP_WOL) && - (if_getcapabilities(ifp) & IFCAP_WOL) != 0) { - if (mask & IFCAP_WOL_MCAST) - if_togglecapenable(ifp, IFCAP_WOL_MCAST); - if (mask & IFCAP_WOL_MAGIC) - if_togglecapenable(ifp, IFCAP_WOL_MAGIC); - } - if (reinit && (if_getdrvflags(ifp) & IFF_DRV_RUNNING)) - lem_init(adapter); - if_vlancap(ifp); - break; - } - - default: - error = ether_ioctl(ifp, command, data); - break; - } - - return (error); -} - - -/********************************************************************* - * Init entry point - * - * This routine is used in two ways. It is used by the stack as - * init entry point in network interface structure. It is also used - * by the driver as a hw/sw initialization routine to get to a - * consistent state. - * - * return 0 on success, positive on failure - **********************************************************************/ - -static void -lem_init_locked(struct adapter *adapter) -{ - if_t ifp = adapter->ifp; - device_t dev = adapter->dev; - u32 pba; - - INIT_DEBUGOUT("lem_init: begin"); - - EM_CORE_LOCK_ASSERT(adapter); - - EM_TX_LOCK(adapter); - lem_stop(adapter); - EM_TX_UNLOCK(adapter); - - /* - * Packet Buffer Allocation (PBA) - * Writing PBA sets the receive portion of the buffer - * the remainder is used for the transmit buffer. - * - * Devices before the 82547 had a Packet Buffer of 64K. - * Default allocation: PBA=48K for Rx, leaving 16K for Tx. - * After the 82547 the buffer was reduced to 40K. - * Default allocation: PBA=30K for Rx, leaving 10K for Tx. - * Note: default does not leave enough room for Jumbo Frame >10k. - */ - switch (adapter->hw.mac.type) { - case e1000_82547: - case e1000_82547_rev_2: /* 82547: Total Packet Buffer is 40K */ - if (adapter->max_frame_size > 8192) - pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */ - else - pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */ - adapter->tx_fifo_head = 0; - adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT; - adapter->tx_fifo_size = - (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT; - break; - default: - /* Devices before 82547 had a Packet Buffer of 64K. */ - if (adapter->max_frame_size > 8192) - pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */ - else - pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */ - } - - INIT_DEBUGOUT1("lem_init: pba=%dK",pba); - E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba); - - /* Get the latest mac address, User can use a LAA */ - bcopy(if_getlladdr(adapter->ifp), adapter->hw.mac.addr, - ETHER_ADDR_LEN); - - /* Put the address into the Receive Address Array */ - e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); - - /* Initialize the hardware */ - if (lem_hardware_init(adapter)) { - device_printf(dev, "Unable to initialize the hardware\n"); - return; - } - lem_update_link_status(adapter); - - /* Setup VLAN support, basic and offload if available */ - E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); - - /* Set hardware offload abilities */ - if_clearhwassist(ifp); - if (adapter->hw.mac.type >= e1000_82543) { - if (if_getcapenable(ifp) & IFCAP_TXCSUM) - if_sethwassistbits(ifp, CSUM_TCP | CSUM_UDP, 0); - } - - /* Configure for OS presence */ - lem_init_manageability(adapter); - - /* Prepare transmit descriptors and buffers */ - lem_setup_transmit_structures(adapter); - lem_initialize_transmit_unit(adapter); - - /* Setup Multicast table */ - lem_set_multi(adapter); - - /* Prepare receive descriptors and buffers */ - if (lem_setup_receive_structures(adapter)) { - device_printf(dev, "Could not setup receive structures\n"); - EM_TX_LOCK(adapter); - lem_stop(adapter); - EM_TX_UNLOCK(adapter); - return; - } - lem_initialize_receive_unit(adapter); - - /* Use real VLAN Filter support? */ - if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) { - if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) - /* Use real VLAN Filter support */ - lem_setup_vlan_hw_support(adapter); - else { - u32 ctrl; - ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); - ctrl |= E1000_CTRL_VME; - E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); - } - } - - /* Don't lose promiscuous settings */ - lem_set_promisc(adapter); - - if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); - - callout_reset(&adapter->timer, hz, lem_local_timer, adapter); - e1000_clear_hw_cntrs_base_generic(&adapter->hw); - -#ifdef DEVICE_POLLING - /* - * Only enable interrupts if we are not polling, make sure - * they are off otherwise. - */ - if (if_getcapenable(ifp) & IFCAP_POLLING) - lem_disable_intr(adapter); - else -#endif /* DEVICE_POLLING */ - lem_enable_intr(adapter); - - /* AMT based hardware can now take control from firmware */ - if (adapter->has_manage && adapter->has_amt) - lem_get_hw_control(adapter); -} - -static void -lem_init(void *arg) -{ - struct adapter *adapter = arg; - - EM_CORE_LOCK(adapter); - lem_init_locked(adapter); - EM_CORE_UNLOCK(adapter); -} - - -#ifdef DEVICE_POLLING -/********************************************************************* - * - * Legacy polling routine - * - *********************************************************************/ -static int -lem_poll(if_t ifp, enum poll_cmd cmd, int count) -{ - struct adapter *adapter = if_getsoftc(ifp); - u32 reg_icr, rx_done = 0; - - EM_CORE_LOCK(adapter); - if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) { - EM_CORE_UNLOCK(adapter); - return (rx_done); - } - - if (cmd == POLL_AND_CHECK_STATUS) { - reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); - if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { - callout_stop(&adapter->timer); - adapter->hw.mac.get_link_status = 1; - lem_update_link_status(adapter); - callout_reset(&adapter->timer, hz, - lem_local_timer, adapter); - } - } - EM_CORE_UNLOCK(adapter); - - lem_rxeof(adapter, count, &rx_done); - - EM_TX_LOCK(adapter); - lem_txeof(adapter); - if(!if_sendq_empty(ifp)) - lem_start_locked(ifp); - EM_TX_UNLOCK(adapter); - return (rx_done); -} -#endif /* DEVICE_POLLING */ - -/********************************************************************* - * - * Legacy Interrupt Service routine - * - *********************************************************************/ -static void -lem_intr(void *arg) -{ - struct adapter *adapter = arg; - if_t ifp = adapter->ifp; - u32 reg_icr; - - - if ((if_getcapenable(ifp) & IFCAP_POLLING) || - ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)) - return; - - EM_CORE_LOCK(adapter); - reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); - if (reg_icr & E1000_ICR_RXO) - adapter->rx_overruns++; - - if ((reg_icr == 0xffffffff) || (reg_icr == 0)) { - EM_CORE_UNLOCK(adapter); - return; - } - - if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { - callout_stop(&adapter->timer); - adapter->hw.mac.get_link_status = 1; - lem_update_link_status(adapter); - /* Deal with TX cruft when link lost */ - lem_tx_purge(adapter); - callout_reset(&adapter->timer, hz, - lem_local_timer, adapter); - EM_CORE_UNLOCK(adapter); - return; - } - - EM_CORE_UNLOCK(adapter); - lem_rxeof(adapter, -1, NULL); - - EM_TX_LOCK(adapter); - lem_txeof(adapter); - if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) && - (!if_sendq_empty(ifp))) - lem_start_locked(ifp); - EM_TX_UNLOCK(adapter); - return; -} - - -static void -lem_handle_link(void *context, int pending) -{ - struct adapter *adapter = context; - if_t ifp = adapter->ifp; - - if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) - return; - - EM_CORE_LOCK(adapter); - callout_stop(&adapter->timer); - lem_update_link_status(adapter); - /* Deal with TX cruft when link lost */ - lem_tx_purge(adapter); - callout_reset(&adapter->timer, hz, lem_local_timer, adapter); - EM_CORE_UNLOCK(adapter); -} - - -/* Combined RX/TX handler, used by Legacy and MSI */ -static void -lem_handle_rxtx(void *context, int pending) -{ - struct adapter *adapter = context; - if_t ifp = adapter->ifp; - - - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { - bool more = lem_rxeof(adapter, adapter->rx_process_limit, NULL); - EM_TX_LOCK(adapter); - lem_txeof(adapter); - if(!if_sendq_empty(ifp)) - lem_start_locked(ifp); - EM_TX_UNLOCK(adapter); - if (more) { - taskqueue_enqueue(adapter->tq, &adapter->rxtx_task); - return; - } - } - - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) - lem_enable_intr(adapter); -} - -/********************************************************************* - * - * Fast Legacy/MSI Combined Interrupt Service routine - * - *********************************************************************/ -static int -lem_irq_fast(void *arg) -{ - struct adapter *adapter = arg; - if_t ifp; - u32 reg_icr; - - ifp = adapter->ifp; - - reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); - - /* Hot eject? */ - if (reg_icr == 0xffffffff) - return FILTER_STRAY; - - /* Definitely not our interrupt. */ - if (reg_icr == 0x0) - return FILTER_STRAY; - - /* - * Mask interrupts until the taskqueue is finished running. This is - * cheap, just assume that it is needed. This also works around the - * MSI message reordering errata on certain systems. - */ - lem_disable_intr(adapter); - taskqueue_enqueue(adapter->tq, &adapter->rxtx_task); - - /* Link status change */ - if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { - adapter->hw.mac.get_link_status = 1; - taskqueue_enqueue(taskqueue_fast, &adapter->link_task); - } - - if (reg_icr & E1000_ICR_RXO) - adapter->rx_overruns++; - return FILTER_HANDLED; -} - - -/********************************************************************* - * - * Media Ioctl callback - * - * This routine is called whenever the user queries the status of - * the interface using ifconfig. - * - **********************************************************************/ -static void -lem_media_status(if_t ifp, struct ifmediareq *ifmr) -{ - struct adapter *adapter = if_getsoftc(ifp); - u_char fiber_type = IFM_1000_SX; - - INIT_DEBUGOUT("lem_media_status: begin"); - - EM_CORE_LOCK(adapter); - lem_update_link_status(adapter); - - ifmr->ifm_status = IFM_AVALID; - ifmr->ifm_active = IFM_ETHER; - - if (!adapter->link_active) { - EM_CORE_UNLOCK(adapter); - return; - } - - ifmr->ifm_status |= IFM_ACTIVE; - - if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || - (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { - if (adapter->hw.mac.type == e1000_82545) - fiber_type = IFM_1000_LX; - ifmr->ifm_active |= fiber_type | IFM_FDX; - } else { - switch (adapter->link_speed) { - case 10: - ifmr->ifm_active |= IFM_10_T; - break; - case 100: - ifmr->ifm_active |= IFM_100_TX; - break; - case 1000: - ifmr->ifm_active |= IFM_1000_T; - break; - } - if (adapter->link_duplex == FULL_DUPLEX) - ifmr->ifm_active |= IFM_FDX; - else - ifmr->ifm_active |= IFM_HDX; - } - EM_CORE_UNLOCK(adapter); -} - -/********************************************************************* - * - * Media Ioctl callback - * - * This routine is called when the user changes speed/duplex using - * media/mediopt option with ifconfig. - * - **********************************************************************/ -static int -lem_media_change(if_t ifp) -{ - struct adapter *adapter = if_getsoftc(ifp); - struct ifmedia *ifm = &adapter->media; - - INIT_DEBUGOUT("lem_media_change: begin"); - - if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) - return (EINVAL); - - EM_CORE_LOCK(adapter); - switch (IFM_SUBTYPE(ifm->ifm_media)) { - case IFM_AUTO: - adapter->hw.mac.autoneg = DO_AUTO_NEG; - adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; - break; - case IFM_1000_LX: - case IFM_1000_SX: - case IFM_1000_T: - adapter->hw.mac.autoneg = DO_AUTO_NEG; - adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; - break; - case IFM_100_TX: - adapter->hw.mac.autoneg = FALSE; - adapter->hw.phy.autoneg_advertised = 0; - if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) - adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; - else - adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; - break; - case IFM_10_T: - adapter->hw.mac.autoneg = FALSE; - adapter->hw.phy.autoneg_advertised = 0; - if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) - adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; - else - adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; - break; - default: - device_printf(adapter->dev, "Unsupported media type\n"); - } - - lem_init_locked(adapter); - EM_CORE_UNLOCK(adapter); - - return (0); -} - -/********************************************************************* - * - * This routine maps the mbufs to tx descriptors. - * - * return 0 on success, positive on failure - **********************************************************************/ - -static int -lem_xmit(struct adapter *adapter, struct mbuf **m_headp) -{ - bus_dma_segment_t segs[EM_MAX_SCATTER]; - bus_dmamap_t map; - struct em_buffer *tx_buffer, *tx_buffer_mapped; - struct e1000_tx_desc *ctxd = NULL; - struct mbuf *m_head; - u32 txd_upper, txd_lower, txd_used, txd_saved; - int error, nsegs, i, j, first, last = 0; - - m_head = *m_headp; - txd_upper = txd_lower = txd_used = txd_saved = 0; - - /* - ** When doing checksum offload, it is critical to - ** make sure the first mbuf has more than header, - ** because that routine expects data to be present. - */ - if ((m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) && - (m_head->m_len < ETHER_HDR_LEN + sizeof(struct ip))) { - m_head = m_pullup(m_head, ETHER_HDR_LEN + sizeof(struct ip)); - *m_headp = m_head; - if (m_head == NULL) - return (ENOBUFS); - } - - /* - * Map the packet for DMA - * - * Capture the first descriptor index, - * this descriptor will have the index - * of the EOP which is the only one that - * now gets a DONE bit writeback. - */ - first = adapter->next_avail_tx_desc; - tx_buffer = &adapter->tx_buffer_area[first]; - tx_buffer_mapped = tx_buffer; - map = tx_buffer->map; - - error = bus_dmamap_load_mbuf_sg(adapter->txtag, map, - *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); - - /* - * There are two types of errors we can (try) to handle: - * - EFBIG means the mbuf chain was too long and bus_dma ran - * out of segments. Defragment the mbuf chain and try again. - * - ENOMEM means bus_dma could not obtain enough bounce buffers - * at this point in time. Defer sending and try again later. - * All other errors, in particular EINVAL, are fatal and prevent the - * mbuf chain from ever going through. Drop it and report error. - */ - if (error == EFBIG) { - struct mbuf *m; - - m = m_collapse(*m_headp, M_NOWAIT, EM_MAX_SCATTER); - if (m == NULL) { - adapter->mbuf_defrag_failed++; - m_freem(*m_headp); - *m_headp = NULL; - return (ENOBUFS); - } - *m_headp = m; - - /* Try it again */ - error = bus_dmamap_load_mbuf_sg(adapter->txtag, map, - *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); - - if (error) { - adapter->no_tx_dma_setup++; - m_freem(*m_headp); - *m_headp = NULL; - return (error); - } - } else if (error != 0) { - adapter->no_tx_dma_setup++; - return (error); - } - - if (adapter->num_tx_desc_avail < (nsegs + 2)) { - adapter->no_tx_desc_avail2++; - bus_dmamap_unload(adapter->txtag, map); - return (ENOBUFS); - } - m_head = *m_headp; - - /* Do hardware assists */ - if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) - lem_transmit_checksum_setup(adapter, m_head, - &txd_upper, &txd_lower); - - i = adapter->next_avail_tx_desc; - if (adapter->pcix_82544) - txd_saved = i; - - /* Set up our transmit descriptors */ - for (j = 0; j < nsegs; j++) { - bus_size_t seg_len; - bus_addr_t seg_addr; - /* If adapter is 82544 and on PCIX bus */ - if(adapter->pcix_82544) { - DESC_ARRAY desc_array; - u32 array_elements, counter; - /* - * Check the Address and Length combination and - * split the data accordingly - */ - array_elements = lem_fill_descriptors(segs[j].ds_addr, - segs[j].ds_len, &desc_array); - for (counter = 0; counter < array_elements; counter++) { - if (txd_used == adapter->num_tx_desc_avail) { - adapter->next_avail_tx_desc = txd_saved; - adapter->no_tx_desc_avail2++; - bus_dmamap_unload(adapter->txtag, map); - return (ENOBUFS); - } - tx_buffer = &adapter->tx_buffer_area[i]; - ctxd = &adapter->tx_desc_base[i]; - ctxd->buffer_addr = htole64( - desc_array.descriptor[counter].address); - ctxd->lower.data = htole32( - (adapter->txd_cmd | txd_lower | (u16) - desc_array.descriptor[counter].length)); - ctxd->upper.data = - htole32((txd_upper)); - last = i; - if (++i == adapter->num_tx_desc) - i = 0; - tx_buffer->m_head = NULL; - tx_buffer->next_eop = -1; - txd_used++; - } - } else { - tx_buffer = &adapter->tx_buffer_area[i]; - ctxd = &adapter->tx_desc_base[i]; - seg_addr = segs[j].ds_addr; - seg_len = segs[j].ds_len; - ctxd->buffer_addr = htole64(seg_addr); - ctxd->lower.data = htole32( - adapter->txd_cmd | txd_lower | seg_len); - ctxd->upper.data = - htole32(txd_upper); - last = i; - if (++i == adapter->num_tx_desc) - i = 0; - tx_buffer->m_head = NULL; - tx_buffer->next_eop = -1; - } - } - - adapter->next_avail_tx_desc = i; - - if (adapter->pcix_82544) - adapter->num_tx_desc_avail -= txd_used; - else - adapter->num_tx_desc_avail -= nsegs; - - if (m_head->m_flags & M_VLANTAG) { - /* Set the vlan id. */ - ctxd->upper.fields.special = - htole16(m_head->m_pkthdr.ether_vtag); - /* Tell hardware to add tag */ - ctxd->lower.data |= htole32(E1000_TXD_CMD_VLE); - } - - tx_buffer->m_head = m_head; - tx_buffer_mapped->map = tx_buffer->map; - tx_buffer->map = map; - bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE); - - /* - * Last Descriptor of Packet - * needs End Of Packet (EOP) - * and Report Status (RS) - */ - ctxd->lower.data |= - htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS); - /* - * Keep track in the first buffer which - * descriptor will be written back - */ - tx_buffer = &adapter->tx_buffer_area[first]; - tx_buffer->next_eop = last; - adapter->watchdog_time = ticks; - - /* - * Advance the Transmit Descriptor Tail (TDT), this tells the E1000 - * that this frame is available to transmit. - */ - bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - -#ifdef NIC_SEND_COMBINING - if (adapter->sc_enable) { - if (adapter->shadow_tdt & MIT_PENDING_INT) { - /* signal intr and data pending */ - adapter->shadow_tdt = MIT_PENDING_TDT | (i & 0xffff); - return (0); - } else { - adapter->shadow_tdt = MIT_PENDING_INT; - } - } -#endif /* NIC_SEND_COMBINING */ - - if (adapter->hw.mac.type == e1000_82547 && - adapter->link_duplex == HALF_DUPLEX) - lem_82547_move_tail(adapter); - else { - E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), i); - if (adapter->hw.mac.type == e1000_82547) - lem_82547_update_fifo_head(adapter, - m_head->m_pkthdr.len); - } - - return (0); -} - -/********************************************************************* - * - * 82547 workaround to avoid controller hang in half-duplex environment. - * The workaround is to avoid queuing a large packet that would span - * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers - * in this case. We do that only when FIFO is quiescent. - * - **********************************************************************/ -static void -lem_82547_move_tail(void *arg) -{ - struct adapter *adapter = arg; - struct e1000_tx_desc *tx_desc; - u16 hw_tdt, sw_tdt, length = 0; - bool eop = 0; - - EM_TX_LOCK_ASSERT(adapter); - - hw_tdt = E1000_READ_REG(&adapter->hw, E1000_TDT(0)); - sw_tdt = adapter->next_avail_tx_desc; - - while (hw_tdt != sw_tdt) { - tx_desc = &adapter->tx_desc_base[hw_tdt]; - length += tx_desc->lower.flags.length; - eop = tx_desc->lower.data & E1000_TXD_CMD_EOP; - if (++hw_tdt == adapter->num_tx_desc) - hw_tdt = 0; - - if (eop) { - if (lem_82547_fifo_workaround(adapter, length)) { - adapter->tx_fifo_wrk_cnt++; - callout_reset(&adapter->tx_fifo_timer, 1, - lem_82547_move_tail, adapter); - break; - } - E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), hw_tdt); - lem_82547_update_fifo_head(adapter, length); - length = 0; - } - } -} - -static int -lem_82547_fifo_workaround(struct adapter *adapter, int len) -{ - int fifo_space, fifo_pkt_len; - - fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR); - - if (adapter->link_duplex == HALF_DUPLEX) { - fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head; - - if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) { - if (lem_82547_tx_fifo_reset(adapter)) - return (0); - else - return (1); - } - } - - return (0); -} - -static void -lem_82547_update_fifo_head(struct adapter *adapter, int len) -{ - int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR); - - /* tx_fifo_head is always 16 byte aligned */ - adapter->tx_fifo_head += fifo_pkt_len; - if (adapter->tx_fifo_head >= adapter->tx_fifo_size) { - adapter->tx_fifo_head -= adapter->tx_fifo_size; - } -} - - -static int -lem_82547_tx_fifo_reset(struct adapter *adapter) -{ - u32 tctl; - - if ((E1000_READ_REG(&adapter->hw, E1000_TDT(0)) == - E1000_READ_REG(&adapter->hw, E1000_TDH(0))) && - (E1000_READ_REG(&adapter->hw, E1000_TDFT) == - E1000_READ_REG(&adapter->hw, E1000_TDFH)) && - (E1000_READ_REG(&adapter->hw, E1000_TDFTS) == - E1000_READ_REG(&adapter->hw, E1000_TDFHS)) && - (E1000_READ_REG(&adapter->hw, E1000_TDFPC) == 0)) { - /* Disable TX unit */ - tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL); - E1000_WRITE_REG(&adapter->hw, E1000_TCTL, - tctl & ~E1000_TCTL_EN); - - /* Reset FIFO pointers */ - E1000_WRITE_REG(&adapter->hw, E1000_TDFT, - adapter->tx_head_addr); - E1000_WRITE_REG(&adapter->hw, E1000_TDFH, - adapter->tx_head_addr); - E1000_WRITE_REG(&adapter->hw, E1000_TDFTS, - adapter->tx_head_addr); - E1000_WRITE_REG(&adapter->hw, E1000_TDFHS, - adapter->tx_head_addr); - - /* Re-enable TX unit */ - E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); - E1000_WRITE_FLUSH(&adapter->hw); - - adapter->tx_fifo_head = 0; - adapter->tx_fifo_reset_cnt++; - - return (TRUE); - } - else { - return (FALSE); - } -} - -static void -lem_set_promisc(struct adapter *adapter) -{ - if_t ifp = adapter->ifp; - u32 reg_rctl; - - reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); - - if (if_getflags(ifp) & IFF_PROMISC) { - reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); - /* Turn this on if you want to see bad packets */ - if (lem_debug_sbp) - reg_rctl |= E1000_RCTL_SBP; - E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); - } else if (if_getflags(ifp) & IFF_ALLMULTI) { - reg_rctl |= E1000_RCTL_MPE; - reg_rctl &= ~E1000_RCTL_UPE; - E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); - } -} - -static void -lem_disable_promisc(struct adapter *adapter) -{ - if_t ifp = adapter->ifp; - u32 reg_rctl; - int mcnt = 0; - - reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); - reg_rctl &= (~E1000_RCTL_UPE); - if (if_getflags(ifp) & IFF_ALLMULTI) - mcnt = MAX_NUM_MULTICAST_ADDRESSES; - else - mcnt = if_multiaddr_count(ifp, MAX_NUM_MULTICAST_ADDRESSES); - - /* Don't disable if in MAX groups */ - if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) - reg_rctl &= (~E1000_RCTL_MPE); - reg_rctl &= (~E1000_RCTL_SBP); - E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); -} - - -/********************************************************************* - * Multicast Update - * - * This routine is called whenever multicast address list is updated. - * - **********************************************************************/ - -static void -lem_set_multi(struct adapter *adapter) -{ - if_t ifp = adapter->ifp; - u32 reg_rctl = 0; - u8 *mta; /* Multicast array memory */ - int mcnt = 0; - - IOCTL_DEBUGOUT("lem_set_multi: begin"); - - mta = adapter->mta; - bzero(mta, sizeof(u8) * ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES); - - if (adapter->hw.mac.type == e1000_82542 && - adapter->hw.revision_id == E1000_REVISION_2) { - reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); - if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) - e1000_pci_clear_mwi(&adapter->hw); - reg_rctl |= E1000_RCTL_RST; - E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); - msec_delay(5); - } - - if_multiaddr_array(ifp, mta, &mcnt, MAX_NUM_MULTICAST_ADDRESSES); - - if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) { - reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); - reg_rctl |= E1000_RCTL_MPE; - E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); - } else - e1000_update_mc_addr_list(&adapter->hw, mta, mcnt); - - if (adapter->hw.mac.type == e1000_82542 && - adapter->hw.revision_id == E1000_REVISION_2) { - reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); - reg_rctl &= ~E1000_RCTL_RST; - E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); - msec_delay(5); - if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) - e1000_pci_set_mwi(&adapter->hw); - } -} - - -/********************************************************************* - * Timer routine - * - * This routine checks for link status and updates statistics. - * - **********************************************************************/ - -static void -lem_local_timer(void *arg) -{ - struct adapter *adapter = arg; - - EM_CORE_LOCK_ASSERT(adapter); - - lem_update_link_status(adapter); - lem_update_stats_counters(adapter); - - lem_smartspeed(adapter); - - /* - * We check the watchdog: the time since - * the last TX descriptor was cleaned. - * This implies a functional TX engine. - */ - if ((adapter->watchdog_check == TRUE) && - (ticks - adapter->watchdog_time > EM_WATCHDOG)) - goto hung; - - callout_reset(&adapter->timer, hz, lem_local_timer, adapter); - return; -hung: - device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); - if_setdrvflagbits(adapter->ifp, 0, IFF_DRV_RUNNING); - adapter->watchdog_events++; - lem_init_locked(adapter); -} - -static void -lem_update_link_status(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - if_t ifp = adapter->ifp; - device_t dev = adapter->dev; - u32 link_check = 0; - - /* Get the cached link value or read phy for real */ - switch (hw->phy.media_type) { - case e1000_media_type_copper: - if (hw->mac.get_link_status) { - /* Do the work to read phy */ - e1000_check_for_link(hw); - link_check = !hw->mac.get_link_status; - if (link_check) /* ESB2 fix */ - e1000_cfg_on_link_up(hw); - } else - link_check = TRUE; - break; - case e1000_media_type_fiber: - e1000_check_for_link(hw); - link_check = (E1000_READ_REG(hw, E1000_STATUS) & - E1000_STATUS_LU); - break; - case e1000_media_type_internal_serdes: - e1000_check_for_link(hw); - link_check = adapter->hw.mac.serdes_has_link; - break; - default: - case e1000_media_type_unknown: - break; - } - - /* Now check for a transition */ - if (link_check && (adapter->link_active == 0)) { - e1000_get_speed_and_duplex(hw, &adapter->link_speed, - &adapter->link_duplex); - if (bootverbose) - device_printf(dev, "Link is up %d Mbps %s\n", - adapter->link_speed, - ((adapter->link_duplex == FULL_DUPLEX) ? - "Full Duplex" : "Half Duplex")); - adapter->link_active = 1; - adapter->smartspeed = 0; - if_setbaudrate(ifp, adapter->link_speed * 1000000); - if_link_state_change(ifp, LINK_STATE_UP); - } else if (!link_check && (adapter->link_active == 1)) { - if_setbaudrate(ifp, 0); - adapter->link_speed = 0; - adapter->link_duplex = 0; - if (bootverbose) - device_printf(dev, "Link is Down\n"); - adapter->link_active = 0; - /* Link down, disable watchdog */ - adapter->watchdog_check = FALSE; - if_link_state_change(ifp, LINK_STATE_DOWN); - } -} - -/********************************************************************* - * - * This routine disables all traffic on the adapter by issuing a - * global reset on the MAC and deallocates TX/RX buffers. - * - * This routine should always be called with BOTH the CORE - * and TX locks. - **********************************************************************/ - -static void -lem_stop(void *arg) -{ - struct adapter *adapter = arg; - if_t ifp = adapter->ifp; - - EM_CORE_LOCK_ASSERT(adapter); - EM_TX_LOCK_ASSERT(adapter); - - INIT_DEBUGOUT("lem_stop: begin"); - - lem_disable_intr(adapter); - callout_stop(&adapter->timer); - callout_stop(&adapter->tx_fifo_timer); - - /* Tell the stack that the interface is no longer active */ - if_setdrvflagbits(ifp, 0, (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)); - - e1000_reset_hw(&adapter->hw); - if (adapter->hw.mac.type >= e1000_82544) - E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); - - e1000_led_off(&adapter->hw); - e1000_cleanup_led(&adapter->hw); -} - - -/********************************************************************* - * - * Determine hardware revision. - * - **********************************************************************/ -static void -lem_identify_hardware(struct adapter *adapter) -{ - device_t dev = adapter->dev; - - /* Make sure our PCI config space has the necessary stuff set */ - pci_enable_busmaster(dev); - adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); - - /* Save off the information about this board */ - adapter->hw.vendor_id = pci_get_vendor(dev); - adapter->hw.device_id = pci_get_device(dev); - adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); - adapter->hw.subsystem_vendor_id = - pci_read_config(dev, PCIR_SUBVEND_0, 2); - adapter->hw.subsystem_device_id = - pci_read_config(dev, PCIR_SUBDEV_0, 2); - - /* Do Shared Code Init and Setup */ - if (e1000_set_mac_type(&adapter->hw)) { - device_printf(dev, "Setup init failure\n"); - return; - } -} - -static int -lem_allocate_pci_resources(struct adapter *adapter) -{ - device_t dev = adapter->dev; - int val, rid, error = E1000_SUCCESS; - - rid = PCIR_BAR(0); - adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, - &rid, RF_ACTIVE); - if (adapter->memory == NULL) { - device_printf(dev, "Unable to allocate bus resource: memory\n"); - return (ENXIO); - } - adapter->osdep.mem_bus_space_tag = - rman_get_bustag(adapter->memory); - adapter->osdep.mem_bus_space_handle = - rman_get_bushandle(adapter->memory); - adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; - - /* Only older adapters use IO mapping */ - if (adapter->hw.mac.type > e1000_82543) { - /* Figure our where our IO BAR is ? */ - for (rid = PCIR_BAR(0); rid < PCIR_CIS;) { - val = pci_read_config(dev, rid, 4); - if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) { - adapter->io_rid = rid; - break; - } - rid += 4; - /* check for 64bit BAR */ - if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT) - rid += 4; - } - if (rid >= PCIR_CIS) { - device_printf(dev, "Unable to locate IO BAR\n"); - return (ENXIO); - } - adapter->ioport = bus_alloc_resource_any(dev, - SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE); - if (adapter->ioport == NULL) { - device_printf(dev, "Unable to allocate bus resource: " - "ioport\n"); - return (ENXIO); - } - adapter->hw.io_base = 0; - adapter->osdep.io_bus_space_tag = - rman_get_bustag(adapter->ioport); - adapter->osdep.io_bus_space_handle = - rman_get_bushandle(adapter->ioport); - } - - adapter->hw.back = &adapter->osdep; - - return (error); -} - -/********************************************************************* - * - * Setup the Legacy or MSI Interrupt handler - * - **********************************************************************/ -int -lem_allocate_irq(struct adapter *adapter) -{ - device_t dev = adapter->dev; - int error, rid = 0; - - /* Manually turn off all interrupts */ - E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); - - /* We allocate a single interrupt resource */ - adapter->res[0] = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); - if (adapter->res[0] == NULL) { - device_printf(dev, "Unable to allocate bus resource: " - "interrupt\n"); - return (ENXIO); - } - - /* Do Legacy setup? */ - if (lem_use_legacy_irq) { - if ((error = bus_setup_intr(dev, adapter->res[0], - INTR_TYPE_NET | INTR_MPSAFE, NULL, lem_intr, adapter, - &adapter->tag[0])) != 0) { - device_printf(dev, - "Failed to register interrupt handler"); - return (error); - } - return (0); - } - - /* - * Use a Fast interrupt and the associated - * deferred processing contexts. - */ - TASK_INIT(&adapter->rxtx_task, 0, lem_handle_rxtx, adapter); - TASK_INIT(&adapter->link_task, 0, lem_handle_link, adapter); - adapter->tq = taskqueue_create_fast("lem_taskq", M_NOWAIT, - taskqueue_thread_enqueue, &adapter->tq); - taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq", - device_get_nameunit(adapter->dev)); - if ((error = bus_setup_intr(dev, adapter->res[0], - INTR_TYPE_NET, lem_irq_fast, NULL, adapter, - &adapter->tag[0])) != 0) { - device_printf(dev, "Failed to register fast interrupt " - "handler: %d\n", error); - taskqueue_free(adapter->tq); - adapter->tq = NULL; - return (error); - } - - return (0); -} - - -static void -lem_free_pci_resources(struct adapter *adapter) -{ - device_t dev = adapter->dev; - - - if (adapter->tag[0] != NULL) { - bus_teardown_intr(dev, adapter->res[0], - adapter->tag[0]); - adapter->tag[0] = NULL; - } - - if (adapter->res[0] != NULL) { - bus_release_resource(dev, SYS_RES_IRQ, - 0, adapter->res[0]); - } - - if (adapter->memory != NULL) - bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(0), adapter->memory); - - if (adapter->ioport != NULL) - bus_release_resource(dev, SYS_RES_IOPORT, - adapter->io_rid, adapter->ioport); -} - - -/********************************************************************* - * - * Initialize the hardware to a configuration - * as specified by the adapter structure. - * - **********************************************************************/ -static int -lem_hardware_init(struct adapter *adapter) -{ - device_t dev = adapter->dev; - u16 rx_buffer_size; - - INIT_DEBUGOUT("lem_hardware_init: begin"); - - /* Issue a global reset */ - e1000_reset_hw(&adapter->hw); - - /* When hardware is reset, fifo_head is also reset */ - adapter->tx_fifo_head = 0; - - /* - * These parameters control the automatic generation (Tx) and - * response (Rx) to Ethernet PAUSE frames. - * - High water mark should allow for at least two frames to be - * received after sending an XOFF. - * - Low water mark works best when it is very near the high water mark. - * This allows the receiver to restart by sending XON when it has - * drained a bit. Here we use an arbitrary value of 1500 which will - * restart after one full frame is pulled from the buffer. There - * could be several smaller frames in the buffer and if so they will - * not trigger the XON until their total number reduces the buffer - * by 1500. - * - The pause time is fairly large at 1000 x 512ns = 512 usec. - */ - rx_buffer_size = ((E1000_READ_REG(&adapter->hw, E1000_PBA) & - 0xffff) << 10 ); - - adapter->hw.fc.high_water = rx_buffer_size - - roundup2(adapter->max_frame_size, 1024); - adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500; - - adapter->hw.fc.pause_time = EM_FC_PAUSE_TIME; - adapter->hw.fc.send_xon = TRUE; - - /* Set Flow control, use the tunable location if sane */ - if ((lem_fc_setting >= 0) && (lem_fc_setting < 4)) - adapter->hw.fc.requested_mode = lem_fc_setting; - else - adapter->hw.fc.requested_mode = e1000_fc_none; - - if (e1000_init_hw(&adapter->hw) < 0) { - device_printf(dev, "Hardware Initialization Failed\n"); - return (EIO); - } - - e1000_check_for_link(&adapter->hw); - - return (0); -} - -/********************************************************************* - * - * Setup networking device structure and register an interface. - * - **********************************************************************/ -static int -lem_setup_interface(device_t dev, struct adapter *adapter) -{ - if_t ifp; - - INIT_DEBUGOUT("lem_setup_interface: begin"); - - ifp = adapter->ifp = if_gethandle(IFT_ETHER); - if (ifp == (void *)NULL) { - device_printf(dev, "can not allocate ifnet structure\n"); - return (-1); - } - if_initname(ifp, device_get_name(dev), device_get_unit(dev)); - if_setinitfn(ifp, lem_init); - if_setsoftc(ifp, adapter); - if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); - if_setioctlfn(ifp, lem_ioctl); - if_setstartfn(ifp, lem_start); - if_setgetcounterfn(ifp, lem_get_counter); - if_setsendqlen(ifp, adapter->num_tx_desc - 1); - if_setsendqready(ifp); - - ether_ifattach(ifp, adapter->hw.mac.addr); - - if_setcapabilities(ifp, 0); - - if (adapter->hw.mac.type >= e1000_82543) { - if_setcapabilitiesbit(ifp, IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM, 0); - if_setcapenablebit(ifp, IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM, 0); - } - - /* - * Tell the upper layer(s) we support long frames. - */ - if_setifheaderlen(ifp, sizeof(struct ether_vlan_header)); - if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU, 0); - if_setcapenablebit(ifp, IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU, 0); - - /* - ** Dont turn this on by default, if vlans are - ** created on another pseudo device (eg. lagg) - ** then vlan events are not passed thru, breaking - ** operation, but with HW FILTER off it works. If - ** using vlans directly on the em driver you can - ** enable this and get full hardware tag filtering. - */ - if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWFILTER, 0); - -#ifdef DEVICE_POLLING - if_setcapabilitiesbit(ifp, IFCAP_POLLING, 0); -#endif - - /* Enable only WOL MAGIC by default */ - if (adapter->wol) { - if_setcapabilitiesbit(ifp, IFCAP_WOL, 0); - if_setcapenablebit(ifp, IFCAP_WOL_MAGIC, 0); - } - - /* - * Specify the media types supported by this adapter and register - * callbacks to update media and link information - */ - ifmedia_init(&adapter->media, IFM_IMASK, - lem_media_change, lem_media_status); - if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || - (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { - u_char fiber_type = IFM_1000_SX; /* default type */ - - if (adapter->hw.mac.type == e1000_82545) - fiber_type = IFM_1000_LX; - ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX, - 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL); - } else { - ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, - 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, - 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, - 0, NULL); - if (adapter->hw.phy.type != e1000_phy_ife) { - ifmedia_add(&adapter->media, - IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); - ifmedia_add(&adapter->media, - IFM_ETHER | IFM_1000_T, 0, NULL); - } - } - ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); - ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); - return (0); -} - - -/********************************************************************* - * - * Workaround for SmartSpeed on 82541 and 82547 controllers - * - **********************************************************************/ -static void -lem_smartspeed(struct adapter *adapter) -{ - u16 phy_tmp; - - if (adapter->link_active || (adapter->hw.phy.type != e1000_phy_igp) || - adapter->hw.mac.autoneg == 0 || - (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0) - return; - - if (adapter->smartspeed == 0) { - /* If Master/Slave config fault is asserted twice, - * we assume back-to-back */ - e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp); - if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT)) - return; - e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp); - if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) { - e1000_read_phy_reg(&adapter->hw, - PHY_1000T_CTRL, &phy_tmp); - if(phy_tmp & CR_1000T_MS_ENABLE) { - phy_tmp &= ~CR_1000T_MS_ENABLE; - e1000_write_phy_reg(&adapter->hw, - PHY_1000T_CTRL, phy_tmp); - adapter->smartspeed++; - if(adapter->hw.mac.autoneg && - !e1000_copper_link_autoneg(&adapter->hw) && - !e1000_read_phy_reg(&adapter->hw, - PHY_CONTROL, &phy_tmp)) { - phy_tmp |= (MII_CR_AUTO_NEG_EN | - MII_CR_RESTART_AUTO_NEG); - e1000_write_phy_reg(&adapter->hw, - PHY_CONTROL, phy_tmp); - } - } - } - return; - } else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) { - /* If still no link, perhaps using 2/3 pair cable */ - e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp); - phy_tmp |= CR_1000T_MS_ENABLE; - e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp); - if(adapter->hw.mac.autoneg && - !e1000_copper_link_autoneg(&adapter->hw) && - !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) { - phy_tmp |= (MII_CR_AUTO_NEG_EN | - MII_CR_RESTART_AUTO_NEG); - e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp); - } - } - /* Restart process after EM_SMARTSPEED_MAX iterations */ - if(adapter->smartspeed++ == EM_SMARTSPEED_MAX) - adapter->smartspeed = 0; -} - - -/* - * Manage DMA'able memory. - */ -static void -lem_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) -{ - if (error) - return; - *(bus_addr_t *) arg = segs[0].ds_addr; -} - -static int -lem_dma_malloc(struct adapter *adapter, bus_size_t size, - struct em_dma_alloc *dma, int mapflags) -{ - int error; - - error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ - EM_DBA_ALIGN, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - size, /* maxsize */ - 1, /* nsegments */ - size, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockarg */ - &dma->dma_tag); - if (error) { - device_printf(adapter->dev, - "%s: bus_dma_tag_create failed: %d\n", - __func__, error); - goto fail_0; - } - - error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, - BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map); - if (error) { - device_printf(adapter->dev, - "%s: bus_dmamem_alloc(%ju) failed: %d\n", - __func__, (uintmax_t)size, error); - goto fail_2; - } - - dma->dma_paddr = 0; - error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, - size, lem_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); - if (error || dma->dma_paddr == 0) { - device_printf(adapter->dev, - "%s: bus_dmamap_load failed: %d\n", - __func__, error); - goto fail_3; - } - - return (0); - -fail_3: - bus_dmamap_unload(dma->dma_tag, dma->dma_map); -fail_2: - bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); - bus_dma_tag_destroy(dma->dma_tag); -fail_0: - dma->dma_tag = NULL; - - return (error); -} - -static void -lem_dma_free(struct adapter *adapter, struct em_dma_alloc *dma) -{ - if (dma->dma_tag == NULL) - return; - if (dma->dma_paddr != 0) { - bus_dmamap_sync(dma->dma_tag, dma->dma_map, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(dma->dma_tag, dma->dma_map); - dma->dma_paddr = 0; - } - if (dma->dma_vaddr != NULL) { - bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); - dma->dma_vaddr = NULL; - } - bus_dma_tag_destroy(dma->dma_tag); - dma->dma_tag = NULL; -} - - -/********************************************************************* - * - * Allocate memory for tx_buffer structures. The tx_buffer stores all - * the information needed to transmit a packet on the wire. - * - **********************************************************************/ -static int -lem_allocate_transmit_structures(struct adapter *adapter) -{ - device_t dev = adapter->dev; - struct em_buffer *tx_buffer; - int error; - - /* - * Create DMA tags for tx descriptors - */ - if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - MCLBYTES * EM_MAX_SCATTER, /* maxsize */ - EM_MAX_SCATTER, /* nsegments */ - MCLBYTES, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockarg */ - &adapter->txtag)) != 0) { - device_printf(dev, "Unable to allocate TX DMA tag\n"); - goto fail; - } - - adapter->tx_buffer_area = malloc(sizeof(struct em_buffer) * - adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); - if (adapter->tx_buffer_area == NULL) { - device_printf(dev, "Unable to allocate tx_buffer memory\n"); - error = ENOMEM; - goto fail; - } - - /* Create the descriptor buffer dma maps */ - for (int i = 0; i < adapter->num_tx_desc; i++) { - tx_buffer = &adapter->tx_buffer_area[i]; - error = bus_dmamap_create(adapter->txtag, 0, &tx_buffer->map); - if (error != 0) { - device_printf(dev, "Unable to create TX DMA map\n"); - goto fail; - } - tx_buffer->next_eop = -1; - } - - return (0); -fail: - lem_free_transmit_structures(adapter); - return (error); -} - -/********************************************************************* - * - * (Re)Initialize transmit structures. - * - **********************************************************************/ -static void -lem_setup_transmit_structures(struct adapter *adapter) -{ - struct em_buffer *tx_buffer; -#ifdef DEV_NETMAP - /* we are already locked */ - struct netmap_adapter *na = netmap_getna(adapter->ifp); - struct netmap_slot *slot = netmap_reset(na, NR_TX, 0, 0); -#endif /* DEV_NETMAP */ - - /* Clear the old ring contents */ - bzero(adapter->tx_desc_base, - (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc); - - /* Free any existing TX buffers */ - for (int i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { - tx_buffer = &adapter->tx_buffer_area[i]; - bus_dmamap_sync(adapter->txtag, tx_buffer->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(adapter->txtag, tx_buffer->map); - m_freem(tx_buffer->m_head); - tx_buffer->m_head = NULL; -#ifdef DEV_NETMAP - if (slot) { - /* the i-th NIC entry goes to slot si */ - int si = netmap_idx_n2k(&na->tx_rings[0], i); - uint64_t paddr; - void *addr; - - addr = PNMB(na, slot + si, &paddr); - adapter->tx_desc_base[i].buffer_addr = htole64(paddr); - /* reload the map for netmap mode */ - netmap_load_map(na, adapter->txtag, tx_buffer->map, addr); - } -#endif /* DEV_NETMAP */ - tx_buffer->next_eop = -1; - } - - /* Reset state */ - adapter->last_hw_offload = 0; - adapter->next_avail_tx_desc = 0; - adapter->next_tx_to_clean = 0; - adapter->num_tx_desc_avail = adapter->num_tx_desc; - - bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - - return; -} - -/********************************************************************* - * - * Enable transmit unit. - * - **********************************************************************/ -static void -lem_initialize_transmit_unit(struct adapter *adapter) -{ - u32 tctl, tipg = 0; - u64 bus_addr; - - INIT_DEBUGOUT("lem_initialize_transmit_unit: begin"); - /* Setup the Base and Length of the Tx Descriptor Ring */ - bus_addr = adapter->txdma.dma_paddr; - E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(0), - adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); - E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(0), - (u32)(bus_addr >> 32)); - E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(0), - (u32)bus_addr); - /* Setup the HW Tx Head and Tail descriptor pointers */ - E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), 0); - E1000_WRITE_REG(&adapter->hw, E1000_TDH(0), 0); - - HW_DEBUGOUT2("Base = %x, Length = %x\n", - E1000_READ_REG(&adapter->hw, E1000_TDBAL(0)), - E1000_READ_REG(&adapter->hw, E1000_TDLEN(0))); - - /* Set the default values for the Tx Inter Packet Gap timer */ - switch (adapter->hw.mac.type) { - case e1000_82542: - tipg = DEFAULT_82542_TIPG_IPGT; - tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; - tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; - break; - default: - if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || - (adapter->hw.phy.media_type == - e1000_media_type_internal_serdes)) - tipg = DEFAULT_82543_TIPG_IPGT_FIBER; - else - tipg = DEFAULT_82543_TIPG_IPGT_COPPER; - tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; - tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; - } - - E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg); - E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value); - if(adapter->hw.mac.type >= e1000_82540) - E1000_WRITE_REG(&adapter->hw, E1000_TADV, - adapter->tx_abs_int_delay.value); - - /* Program the Transmit Control Register */ - tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL); - tctl &= ~E1000_TCTL_CT; - tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | - (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); - - /* This write will effectively turn on the transmit unit. */ - E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); - - /* Setup Transmit Descriptor Base Settings */ - adapter->txd_cmd = E1000_TXD_CMD_IFCS; - - if (adapter->tx_int_delay.value > 0) - adapter->txd_cmd |= E1000_TXD_CMD_IDE; -} - -/********************************************************************* - * - * Free all transmit related data structures. - * - **********************************************************************/ -static void -lem_free_transmit_structures(struct adapter *adapter) -{ - struct em_buffer *tx_buffer; - - INIT_DEBUGOUT("free_transmit_structures: begin"); - - if (adapter->tx_buffer_area != NULL) { - for (int i = 0; i < adapter->num_tx_desc; i++) { - tx_buffer = &adapter->tx_buffer_area[i]; - if (tx_buffer->m_head != NULL) { - bus_dmamap_sync(adapter->txtag, tx_buffer->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(adapter->txtag, - tx_buffer->map); - m_freem(tx_buffer->m_head); - tx_buffer->m_head = NULL; - } else if (tx_buffer->map != NULL) - bus_dmamap_unload(adapter->txtag, - tx_buffer->map); - if (tx_buffer->map != NULL) { - bus_dmamap_destroy(adapter->txtag, - tx_buffer->map); - tx_buffer->map = NULL; - } - } - } - if (adapter->tx_buffer_area != NULL) { - free(adapter->tx_buffer_area, M_DEVBUF); - adapter->tx_buffer_area = NULL; - } - if (adapter->txtag != NULL) { - bus_dma_tag_destroy(adapter->txtag); - adapter->txtag = NULL; - } -} - -/********************************************************************* - * - * The offload context needs to be set when we transfer the first - * packet of a particular protocol (TCP/UDP). This routine has been - * enhanced to deal with inserted VLAN headers, and IPV6 (not complete) - * - * Added back the old method of keeping the current context type - * and not setting if unnecessary, as this is reported to be a - * big performance win. -jfv - **********************************************************************/ -static void -lem_transmit_checksum_setup(struct adapter *adapter, struct mbuf *mp, - u32 *txd_upper, u32 *txd_lower) -{ - struct e1000_context_desc *TXD = NULL; - struct em_buffer *tx_buffer; - struct ether_vlan_header *eh; - struct ip *ip = NULL; - struct ip6_hdr *ip6; - int curr_txd, ehdrlen; - u32 cmd, hdr_len, ip_hlen; - u16 etype; - u8 ipproto; - - - cmd = hdr_len = ipproto = 0; - *txd_upper = *txd_lower = 0; - curr_txd = adapter->next_avail_tx_desc; - - /* - * Determine where frame payload starts. - * Jump over vlan headers if already present, - * helpful for QinQ too. - */ - eh = mtod(mp, struct ether_vlan_header *); - if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { - etype = ntohs(eh->evl_proto); - ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; - } else { - etype = ntohs(eh->evl_encap_proto); - ehdrlen = ETHER_HDR_LEN; - } - - /* - * We only support TCP/UDP for IPv4 and IPv6 for the moment. - * TODO: Support SCTP too when it hits the tree. - */ - switch (etype) { - case ETHERTYPE_IP: - ip = (struct ip *)(mp->m_data + ehdrlen); - ip_hlen = ip->ip_hl << 2; - - /* Setup of IP header checksum. */ - if (mp->m_pkthdr.csum_flags & CSUM_IP) { - /* - * Start offset for header checksum calculation. - * End offset for header checksum calculation. - * Offset of place to put the checksum. - */ - TXD = (struct e1000_context_desc *) - &adapter->tx_desc_base[curr_txd]; - TXD->lower_setup.ip_fields.ipcss = ehdrlen; - TXD->lower_setup.ip_fields.ipcse = - htole16(ehdrlen + ip_hlen); - TXD->lower_setup.ip_fields.ipcso = - ehdrlen + offsetof(struct ip, ip_sum); - cmd |= E1000_TXD_CMD_IP; - *txd_upper |= E1000_TXD_POPTS_IXSM << 8; - } - - hdr_len = ehdrlen + ip_hlen; - ipproto = ip->ip_p; - - break; - case ETHERTYPE_IPV6: - ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); - ip_hlen = sizeof(struct ip6_hdr); /* XXX: No header stacking. */ - - /* IPv6 doesn't have a header checksum. */ - - hdr_len = ehdrlen + ip_hlen; - ipproto = ip6->ip6_nxt; - break; - - default: - return; - } - - switch (ipproto) { - case IPPROTO_TCP: - if (mp->m_pkthdr.csum_flags & CSUM_TCP) { - *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; - *txd_upper |= E1000_TXD_POPTS_TXSM << 8; - /* no need for context if already set */ - if (adapter->last_hw_offload == CSUM_TCP) - return; - adapter->last_hw_offload = CSUM_TCP; - /* - * Start offset for payload checksum calculation. - * End offset for payload checksum calculation. - * Offset of place to put the checksum. - */ - TXD = (struct e1000_context_desc *) - &adapter->tx_desc_base[curr_txd]; - TXD->upper_setup.tcp_fields.tucss = hdr_len; - TXD->upper_setup.tcp_fields.tucse = htole16(0); - TXD->upper_setup.tcp_fields.tucso = - hdr_len + offsetof(struct tcphdr, th_sum); - cmd |= E1000_TXD_CMD_TCP; - } - break; - case IPPROTO_UDP: - { - if (mp->m_pkthdr.csum_flags & CSUM_UDP) { - *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; - *txd_upper |= E1000_TXD_POPTS_TXSM << 8; - /* no need for context if already set */ - if (adapter->last_hw_offload == CSUM_UDP) - return; - adapter->last_hw_offload = CSUM_UDP; - /* - * Start offset for header checksum calculation. - * End offset for header checksum calculation. - * Offset of place to put the checksum. - */ - TXD = (struct e1000_context_desc *) - &adapter->tx_desc_base[curr_txd]; - TXD->upper_setup.tcp_fields.tucss = hdr_len; - TXD->upper_setup.tcp_fields.tucse = htole16(0); - TXD->upper_setup.tcp_fields.tucso = - hdr_len + offsetof(struct udphdr, uh_sum); - } - /* Fall Thru */ - } - default: - break; - } - - if (TXD == NULL) - return; - TXD->tcp_seg_setup.data = htole32(0); - TXD->cmd_and_length = - htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd); - tx_buffer = &adapter->tx_buffer_area[curr_txd]; - tx_buffer->m_head = NULL; - tx_buffer->next_eop = -1; - - if (++curr_txd == adapter->num_tx_desc) - curr_txd = 0; - - adapter->num_tx_desc_avail--; - adapter->next_avail_tx_desc = curr_txd; -} - - -/********************************************************************** - * - * Examine each tx_buffer in the used queue. If the hardware is done - * processing the packet then free associated resources. The - * tx_buffer is put back on the free queue. - * - **********************************************************************/ -static void -lem_txeof(struct adapter *adapter) -{ - int first, last, done, num_avail; - struct em_buffer *tx_buffer; - struct e1000_tx_desc *tx_desc, *eop_desc; - if_t ifp = adapter->ifp; - - EM_TX_LOCK_ASSERT(adapter); - -#ifdef DEV_NETMAP - if (netmap_tx_irq(ifp, 0)) - return; -#endif /* DEV_NETMAP */ - if (adapter->num_tx_desc_avail == adapter->num_tx_desc) - return; - - num_avail = adapter->num_tx_desc_avail; - first = adapter->next_tx_to_clean; - tx_desc = &adapter->tx_desc_base[first]; - tx_buffer = &adapter->tx_buffer_area[first]; - last = tx_buffer->next_eop; - eop_desc = &adapter->tx_desc_base[last]; - - /* - * What this does is get the index of the - * first descriptor AFTER the EOP of the - * first packet, that way we can do the - * simple comparison on the inner while loop. - */ - if (++last == adapter->num_tx_desc) - last = 0; - done = last; - - bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, - BUS_DMASYNC_POSTREAD); - - while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) { - /* We clean the range of the packet */ - while (first != done) { - tx_desc->upper.data = 0; - tx_desc->lower.data = 0; - tx_desc->buffer_addr = 0; - ++num_avail; - - if (tx_buffer->m_head) { - if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); - bus_dmamap_sync(adapter->txtag, - tx_buffer->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(adapter->txtag, - tx_buffer->map); - - m_freem(tx_buffer->m_head); - tx_buffer->m_head = NULL; - } - tx_buffer->next_eop = -1; - adapter->watchdog_time = ticks; - - if (++first == adapter->num_tx_desc) - first = 0; - - tx_buffer = &adapter->tx_buffer_area[first]; - tx_desc = &adapter->tx_desc_base[first]; - } - /* See if we can continue to the next packet */ - last = tx_buffer->next_eop; - if (last != -1) { - eop_desc = &adapter->tx_desc_base[last]; - /* Get new done point */ - if (++last == adapter->num_tx_desc) last = 0; - done = last; - } else - break; - } - bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - - adapter->next_tx_to_clean = first; - adapter->num_tx_desc_avail = num_avail; - -#ifdef NIC_SEND_COMBINING - if ((adapter->shadow_tdt & MIT_PENDING_TDT) == MIT_PENDING_TDT) { - /* a tdt write is pending, do it */ - E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), - 0xffff & adapter->shadow_tdt); - adapter->shadow_tdt = MIT_PENDING_INT; - } else { - adapter->shadow_tdt = 0; // disable - } -#endif /* NIC_SEND_COMBINING */ - /* - * If we have enough room, clear IFF_DRV_OACTIVE to - * tell the stack that it is OK to send packets. - * If there are no pending descriptors, clear the watchdog. - */ - if (adapter->num_tx_desc_avail > EM_TX_CLEANUP_THRESHOLD) { - if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); - if (adapter->num_tx_desc_avail == adapter->num_tx_desc) { - adapter->watchdog_check = FALSE; - return; - } - } -} - -/********************************************************************* - * - * When Link is lost sometimes there is work still in the TX ring - * which may result in a watchdog, rather than allow that we do an - * attempted cleanup and then reinit here. Note that this has been - * seens mostly with fiber adapters. - * - **********************************************************************/ -static void -lem_tx_purge(struct adapter *adapter) -{ - if ((!adapter->link_active) && (adapter->watchdog_check)) { - EM_TX_LOCK(adapter); - lem_txeof(adapter); - EM_TX_UNLOCK(adapter); - if (adapter->watchdog_check) /* Still outstanding? */ - lem_init_locked(adapter); - } -} - -/********************************************************************* - * - * Get a buffer from system mbuf buffer pool. - * - **********************************************************************/ -static int -lem_get_buf(struct adapter *adapter, int i) -{ - struct mbuf *m; - bus_dma_segment_t segs[1]; - bus_dmamap_t map; - struct em_buffer *rx_buffer; - int error, nsegs; - - m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); - if (m == NULL) { - adapter->mbuf_cluster_failed++; - return (ENOBUFS); - } - m->m_len = m->m_pkthdr.len = MCLBYTES; - - if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN)) - m_adj(m, ETHER_ALIGN); - - /* - * Using memory from the mbuf cluster pool, invoke the - * bus_dma machinery to arrange the memory mapping. - */ - error = bus_dmamap_load_mbuf_sg(adapter->rxtag, - adapter->rx_sparemap, m, segs, &nsegs, BUS_DMA_NOWAIT); - if (error != 0) { - m_free(m); - return (error); - } - - /* If nsegs is wrong then the stack is corrupt. */ - KASSERT(nsegs == 1, ("Too many segments returned!")); - - rx_buffer = &adapter->rx_buffer_area[i]; - if (rx_buffer->m_head != NULL) - bus_dmamap_unload(adapter->rxtag, rx_buffer->map); - - map = rx_buffer->map; - rx_buffer->map = adapter->rx_sparemap; - adapter->rx_sparemap = map; - bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD); - rx_buffer->m_head = m; - - adapter->rx_desc_base[i].buffer_addr = htole64(segs[0].ds_addr); - return (0); -} - -/********************************************************************* - * - * Allocate memory for rx_buffer structures. Since we use one - * rx_buffer per received packet, the maximum number of rx_buffer's - * that we'll need is equal to the number of receive descriptors - * that we've allocated. - * - **********************************************************************/ -static int -lem_allocate_receive_structures(struct adapter *adapter) -{ - device_t dev = adapter->dev; - struct em_buffer *rx_buffer; - int i, error; - - adapter->rx_buffer_area = malloc(sizeof(struct em_buffer) * - adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); - if (adapter->rx_buffer_area == NULL) { - device_printf(dev, "Unable to allocate rx_buffer memory\n"); - return (ENOMEM); - } - - error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - MCLBYTES, /* maxsize */ - 1, /* nsegments */ - MCLBYTES, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockarg */ - &adapter->rxtag); - if (error) { - device_printf(dev, "%s: bus_dma_tag_create failed %d\n", - __func__, error); - goto fail; - } - - /* Create the spare map (used by getbuf) */ - error = bus_dmamap_create(adapter->rxtag, 0, &adapter->rx_sparemap); - if (error) { - device_printf(dev, "%s: bus_dmamap_create failed: %d\n", - __func__, error); - goto fail; - } - - rx_buffer = adapter->rx_buffer_area; - for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { - error = bus_dmamap_create(adapter->rxtag, 0, &rx_buffer->map); - if (error) { - device_printf(dev, "%s: bus_dmamap_create failed: %d\n", - __func__, error); - goto fail; - } - } - - return (0); - -fail: - lem_free_receive_structures(adapter); - return (error); -} - -/********************************************************************* - * - * (Re)initialize receive structures. - * - **********************************************************************/ -static int -lem_setup_receive_structures(struct adapter *adapter) -{ - struct em_buffer *rx_buffer; - int i, error; -#ifdef DEV_NETMAP - /* we are already under lock */ - struct netmap_adapter *na = netmap_getna(adapter->ifp); - struct netmap_slot *slot = netmap_reset(na, NR_RX, 0, 0); -#endif - - /* Reset descriptor ring */ - bzero(adapter->rx_desc_base, - (sizeof(struct e1000_rx_desc)) * adapter->num_rx_desc); - - /* Free current RX buffers. */ - rx_buffer = adapter->rx_buffer_area; - for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { - if (rx_buffer->m_head != NULL) { - bus_dmamap_sync(adapter->rxtag, rx_buffer->map, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(adapter->rxtag, rx_buffer->map); - m_freem(rx_buffer->m_head); - rx_buffer->m_head = NULL; - } - } - - /* Allocate new ones. */ - for (i = 0; i < adapter->num_rx_desc; i++) { -#ifdef DEV_NETMAP - if (slot) { - /* the i-th NIC entry goes to slot si */ - int si = netmap_idx_n2k(&na->rx_rings[0], i); - uint64_t paddr; - void *addr; - - addr = PNMB(na, slot + si, &paddr); - netmap_load_map(na, adapter->rxtag, rx_buffer->map, addr); - /* Update descriptor */ - adapter->rx_desc_base[i].buffer_addr = htole64(paddr); - continue; - } -#endif /* DEV_NETMAP */ - error = lem_get_buf(adapter, i); - if (error) - return (error); - } - - /* Setup our descriptor pointers */ - adapter->next_rx_desc_to_check = 0; - bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - - return (0); -} - -/********************************************************************* - * - * Enable receive unit. - * - **********************************************************************/ - -static void -lem_initialize_receive_unit(struct adapter *adapter) -{ - if_t ifp = adapter->ifp; - u64 bus_addr; - u32 rctl, rxcsum; - - INIT_DEBUGOUT("lem_initialize_receive_unit: begin"); - - /* - * Make sure receives are disabled while setting - * up the descriptor ring - */ - rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); - E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); - - if (adapter->hw.mac.type >= e1000_82540) { - E1000_WRITE_REG(&adapter->hw, E1000_RADV, - adapter->rx_abs_int_delay.value); - /* - * Set the interrupt throttling rate. Value is calculated - * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) - */ - E1000_WRITE_REG(&adapter->hw, E1000_ITR, DEFAULT_ITR); - } - - /* Setup the Base and Length of the Rx Descriptor Ring */ - bus_addr = adapter->rxdma.dma_paddr; - E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(0), - adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); - E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(0), - (u32)(bus_addr >> 32)); - E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(0), - (u32)bus_addr); - - /* Setup the Receive Control Register */ - rctl &= ~(3 << E1000_RCTL_MO_SHIFT); - rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO | - E1000_RCTL_RDMTS_HALF | - (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT); - - /* Make sure VLAN Filters are off */ - rctl &= ~E1000_RCTL_VFE; - - if (e1000_tbi_sbp_enabled_82543(&adapter->hw)) - rctl |= E1000_RCTL_SBP; - else - rctl &= ~E1000_RCTL_SBP; - - switch (adapter->rx_buffer_len) { - default: - case 2048: - rctl |= E1000_RCTL_SZ_2048; - break; - case 4096: - rctl |= E1000_RCTL_SZ_4096 | - E1000_RCTL_BSEX | E1000_RCTL_LPE; - break; - case 8192: - rctl |= E1000_RCTL_SZ_8192 | - E1000_RCTL_BSEX | E1000_RCTL_LPE; - break; - case 16384: - rctl |= E1000_RCTL_SZ_16384 | - E1000_RCTL_BSEX | E1000_RCTL_LPE; - break; - } - - if (if_getmtu(ifp) > ETHERMTU) - rctl |= E1000_RCTL_LPE; - else - rctl &= ~E1000_RCTL_LPE; - - /* Enable 82543 Receive Checksum Offload for TCP and UDP */ - if ((adapter->hw.mac.type >= e1000_82543) && - (if_getcapenable(ifp) & IFCAP_RXCSUM)) { - rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM); - rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL); - E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum); - } - - /* Enable Receives */ - E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); - - /* - * Setup the HW Rx Head and - * Tail Descriptor Pointers - */ - E1000_WRITE_REG(&adapter->hw, E1000_RDH(0), 0); - rctl = adapter->num_rx_desc - 1; /* default RDT value */ -#ifdef DEV_NETMAP - /* preserve buffers already made available to clients */ - if (if_getcapenable(ifp) & IFCAP_NETMAP) { - struct netmap_adapter *na = netmap_getna(adapter->ifp); - rctl -= nm_kr_rxspace(&na->rx_rings[0]); - } -#endif /* DEV_NETMAP */ - E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), rctl); - - return; -} - -/********************************************************************* - * - * Free receive related data structures. - * - **********************************************************************/ -static void -lem_free_receive_structures(struct adapter *adapter) -{ - struct em_buffer *rx_buffer; - int i; - - INIT_DEBUGOUT("free_receive_structures: begin"); - - if (adapter->rx_sparemap) { - bus_dmamap_destroy(adapter->rxtag, adapter->rx_sparemap); - adapter->rx_sparemap = NULL; - } - - /* Cleanup any existing buffers */ - if (adapter->rx_buffer_area != NULL) { - rx_buffer = adapter->rx_buffer_area; - for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { - if (rx_buffer->m_head != NULL) { - bus_dmamap_sync(adapter->rxtag, rx_buffer->map, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(adapter->rxtag, - rx_buffer->map); - m_freem(rx_buffer->m_head); - rx_buffer->m_head = NULL; - } else if (rx_buffer->map != NULL) - bus_dmamap_unload(adapter->rxtag, - rx_buffer->map); - if (rx_buffer->map != NULL) { - bus_dmamap_destroy(adapter->rxtag, - rx_buffer->map); - rx_buffer->map = NULL; - } - } - } - - if (adapter->rx_buffer_area != NULL) { - free(adapter->rx_buffer_area, M_DEVBUF); - adapter->rx_buffer_area = NULL; - } - - if (adapter->rxtag != NULL) { - bus_dma_tag_destroy(adapter->rxtag); - adapter->rxtag = NULL; - } -} - -/********************************************************************* - * - * This routine executes in interrupt context. It replenishes - * the mbufs in the descriptor and sends data which has been - * dma'ed into host memory to upper layer. - * - * We loop at most count times if count is > 0, or until done if - * count < 0. - * - * For polling we also now return the number of cleaned packets - *********************************************************************/ -static bool -lem_rxeof(struct adapter *adapter, int count, int *done) -{ - if_t ifp = adapter->ifp; - struct mbuf *mp; - u8 status = 0, accept_frame = 0, eop = 0; - u16 len, desc_len, prev_len_adj; - int i, rx_sent = 0; - struct e1000_rx_desc *current_desc; - -#ifdef BATCH_DISPATCH - struct mbuf *mh = NULL, *mt = NULL; -#endif /* BATCH_DISPATCH */ - EM_RX_LOCK(adapter); - -#ifdef BATCH_DISPATCH - batch_again: -#endif /* BATCH_DISPATCH */ - i = adapter->next_rx_desc_to_check; - current_desc = &adapter->rx_desc_base[i]; - bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, - BUS_DMASYNC_POSTREAD); - -#ifdef DEV_NETMAP - if (netmap_rx_irq(ifp, 0, &rx_sent)) { - EM_RX_UNLOCK(adapter); - return (FALSE); - } -#endif /* DEV_NETMAP */ - - if (!((current_desc->status) & E1000_RXD_STAT_DD)) { - if (done != NULL) - *done = rx_sent; - EM_RX_UNLOCK(adapter); - return (FALSE); - } - - while (count != 0 && if_getdrvflags(ifp) & IFF_DRV_RUNNING) { - struct mbuf *m = NULL; - - status = current_desc->status; - if ((status & E1000_RXD_STAT_DD) == 0) { - break; - } - - mp = adapter->rx_buffer_area[i].m_head; - /* - * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT - * needs to access the last received byte in the mbuf. - */ - bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map, - BUS_DMASYNC_POSTREAD); - - accept_frame = 1; - prev_len_adj = 0; - desc_len = le16toh(current_desc->length); - if (status & E1000_RXD_STAT_EOP) { - count--; - eop = 1; - if (desc_len < ETHER_CRC_LEN) { - len = 0; - prev_len_adj = ETHER_CRC_LEN - desc_len; - } else - len = desc_len - ETHER_CRC_LEN; - } else { - eop = 0; - len = desc_len; - } - - if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) { - u8 last_byte; - u32 pkt_len = desc_len; - - if (adapter->fmp != NULL) - pkt_len += adapter->fmp->m_pkthdr.len; - - last_byte = *(mtod(mp, caddr_t) + desc_len - 1); - if (TBI_ACCEPT(&adapter->hw, status, - current_desc->errors, pkt_len, last_byte, - adapter->min_frame_size, adapter->max_frame_size)) { - e1000_tbi_adjust_stats_82543(&adapter->hw, - &adapter->stats, pkt_len, - adapter->hw.mac.addr, - adapter->max_frame_size); - if (len > 0) - len--; - } else - accept_frame = 0; - } - - if (accept_frame) { - if (lem_get_buf(adapter, i) != 0) { - if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); - goto discard; - } - - /* Assign correct length to the current fragment */ - mp->m_len = len; - - if (adapter->fmp == NULL) { - mp->m_pkthdr.len = len; - adapter->fmp = mp; /* Store the first mbuf */ - adapter->lmp = mp; - } else { - /* Chain mbuf's together */ - mp->m_flags &= ~M_PKTHDR; - /* - * Adjust length of previous mbuf in chain if - * we received less than 4 bytes in the last - * descriptor. - */ - if (prev_len_adj > 0) { - adapter->lmp->m_len -= prev_len_adj; - adapter->fmp->m_pkthdr.len -= - prev_len_adj; - } - adapter->lmp->m_next = mp; - adapter->lmp = adapter->lmp->m_next; - adapter->fmp->m_pkthdr.len += len; - } - - if (eop) { - if_setrcvif(adapter->fmp, ifp); - if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); - lem_receive_checksum(adapter, current_desc, - adapter->fmp); -#ifndef __NO_STRICT_ALIGNMENT - if (adapter->max_frame_size > - (MCLBYTES - ETHER_ALIGN) && - lem_fixup_rx(adapter) != 0) - goto skip; -#endif - if (status & E1000_RXD_STAT_VP) { - adapter->fmp->m_pkthdr.ether_vtag = - le16toh(current_desc->special); - adapter->fmp->m_flags |= M_VLANTAG; - } -#ifndef __NO_STRICT_ALIGNMENT -skip: -#endif - m = adapter->fmp; - adapter->fmp = NULL; - adapter->lmp = NULL; - } - } else { - adapter->dropped_pkts++; -discard: - /* Reuse loaded DMA map and just update mbuf chain */ - mp = adapter->rx_buffer_area[i].m_head; - mp->m_len = mp->m_pkthdr.len = MCLBYTES; - mp->m_data = mp->m_ext.ext_buf; - mp->m_next = NULL; - if (adapter->max_frame_size <= - (MCLBYTES - ETHER_ALIGN)) - m_adj(mp, ETHER_ALIGN); - if (adapter->fmp != NULL) { - m_freem(adapter->fmp); - adapter->fmp = NULL; - adapter->lmp = NULL; - } - m = NULL; - } - - /* Zero out the receive descriptors status. */ - current_desc->status = 0; - bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - - /* Advance our pointers to the next descriptor. */ - if (++i == adapter->num_rx_desc) - i = 0; - /* Call into the stack */ - if (m != NULL) { -#ifdef BATCH_DISPATCH - if (adapter->batch_enable) { - if (mh == NULL) - mh = mt = m; - else - mt->m_nextpkt = m; - mt = m; - m->m_nextpkt = NULL; - rx_sent++; - current_desc = &adapter->rx_desc_base[i]; - continue; - } -#endif /* BATCH_DISPATCH */ - adapter->next_rx_desc_to_check = i; - EM_RX_UNLOCK(adapter); - if_input(ifp, m); - EM_RX_LOCK(adapter); - rx_sent++; - i = adapter->next_rx_desc_to_check; - } - current_desc = &adapter->rx_desc_base[i]; - } - adapter->next_rx_desc_to_check = i; -#ifdef BATCH_DISPATCH - if (mh) { - EM_RX_UNLOCK(adapter); - while ( (mt = mh) != NULL) { - mh = mh->m_nextpkt; - mt->m_nextpkt = NULL; - if_input(ifp, mt); - } - EM_RX_LOCK(adapter); - i = adapter->next_rx_desc_to_check; /* in case of interrupts */ - if (count > 0) - goto batch_again; - } -#endif /* BATCH_DISPATCH */ - - /* Advance the E1000's Receive Queue #0 "Tail Pointer". */ - if (--i < 0) - i = adapter->num_rx_desc - 1; - E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), i); - if (done != NULL) - *done = rx_sent; - EM_RX_UNLOCK(adapter); - return ((status & E1000_RXD_STAT_DD) ? TRUE : FALSE); -} - -#ifndef __NO_STRICT_ALIGNMENT -/* - * When jumbo frames are enabled we should realign entire payload on - * architecures with strict alignment. This is serious design mistake of 8254x - * as it nullifies DMA operations. 8254x just allows RX buffer size to be - * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its - * payload. On architecures without strict alignment restrictions 8254x still - * performs unaligned memory access which would reduce the performance too. - * To avoid copying over an entire frame to align, we allocate a new mbuf and - * copy ethernet header to the new mbuf. The new mbuf is prepended into the - * existing mbuf chain. - * - * Be aware, best performance of the 8254x is achieved only when jumbo frame is - * not used at all on architectures with strict alignment. - */ -static int -lem_fixup_rx(struct adapter *adapter) -{ - struct mbuf *m, *n; - int error; - - error = 0; - m = adapter->fmp; - if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) { - bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len); - m->m_data += ETHER_HDR_LEN; - } else { - MGETHDR(n, M_NOWAIT, MT_DATA); - if (n != NULL) { - bcopy(m->m_data, n->m_data, ETHER_HDR_LEN); - m->m_data += ETHER_HDR_LEN; - m->m_len -= ETHER_HDR_LEN; - n->m_len = ETHER_HDR_LEN; - M_MOVE_PKTHDR(n, m); - n->m_next = m; - adapter->fmp = n; - } else { - adapter->dropped_pkts++; - m_freem(adapter->fmp); - adapter->fmp = NULL; - error = ENOMEM; - } - } - - return (error); -} -#endif - -/********************************************************************* - * - * Verify that the hardware indicated that the checksum is valid. - * Inform the stack about the status of checksum so that stack - * doesn't spend time verifying the checksum. - * - *********************************************************************/ -static void -lem_receive_checksum(struct adapter *adapter, - struct e1000_rx_desc *rx_desc, struct mbuf *mp) -{ - /* 82543 or newer only */ - if ((adapter->hw.mac.type < e1000_82543) || - /* Ignore Checksum bit is set */ - (rx_desc->status & E1000_RXD_STAT_IXSM)) { - mp->m_pkthdr.csum_flags = 0; - return; - } - - if (rx_desc->status & E1000_RXD_STAT_IPCS) { - /* Did it pass? */ - if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) { - /* IP Checksum Good */ - mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; - mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; - - } else { - mp->m_pkthdr.csum_flags = 0; - } - } - - if (rx_desc->status & E1000_RXD_STAT_TCPCS) { - /* Did it pass? */ - if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) { - mp->m_pkthdr.csum_flags |= - (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); - mp->m_pkthdr.csum_data = htons(0xffff); - } - } -} - -/* - * This routine is run via an vlan - * config EVENT - */ -static void -lem_register_vlan(void *arg, if_t ifp, u16 vtag) -{ - struct adapter *adapter = if_getsoftc(ifp); - u32 index, bit; - - if (if_getsoftc(ifp) != arg) /* Not our event */ - return; - - if ((vtag == 0) || (vtag > 4095)) /* Invalid ID */ - return; - - EM_CORE_LOCK(adapter); - index = (vtag >> 5) & 0x7F; - bit = vtag & 0x1F; - adapter->shadow_vfta[index] |= (1 << bit); - ++adapter->num_vlans; - /* Re-init to load the changes */ - if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) - lem_init_locked(adapter); - EM_CORE_UNLOCK(adapter); -} - -/* - * This routine is run via an vlan - * unconfig EVENT - */ -static void -lem_unregister_vlan(void *arg, if_t ifp, u16 vtag) -{ - struct adapter *adapter = if_getsoftc(ifp); - u32 index, bit; - - if (if_getsoftc(ifp) != arg) - return; - - if ((vtag == 0) || (vtag > 4095)) /* Invalid */ - return; - - EM_CORE_LOCK(adapter); - index = (vtag >> 5) & 0x7F; - bit = vtag & 0x1F; - adapter->shadow_vfta[index] &= ~(1 << bit); - --adapter->num_vlans; - /* Re-init to load the changes */ - if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) - lem_init_locked(adapter); - EM_CORE_UNLOCK(adapter); -} - -static void -lem_setup_vlan_hw_support(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - u32 reg; - - /* - ** We get here thru init_locked, meaning - ** a soft reset, this has already cleared - ** the VFTA and other state, so if there - ** have been no vlan's registered do nothing. - */ - if (adapter->num_vlans == 0) - return; - - /* - ** A soft reset zero's out the VFTA, so - ** we need to repopulate it now. - */ - for (int i = 0; i < EM_VFTA_SIZE; i++) - if (adapter->shadow_vfta[i] != 0) - E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, - i, adapter->shadow_vfta[i]); - - reg = E1000_READ_REG(hw, E1000_CTRL); - reg |= E1000_CTRL_VME; - E1000_WRITE_REG(hw, E1000_CTRL, reg); - - /* Enable the Filter Table */ - reg = E1000_READ_REG(hw, E1000_RCTL); - reg &= ~E1000_RCTL_CFIEN; - reg |= E1000_RCTL_VFE; - E1000_WRITE_REG(hw, E1000_RCTL, reg); -} - -static void -lem_enable_intr(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - u32 ims_mask = IMS_ENABLE_MASK; - - E1000_WRITE_REG(hw, E1000_IMS, ims_mask); -} - -static void -lem_disable_intr(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - - E1000_WRITE_REG(hw, E1000_IMC, 0xffffffff); -} - -/* - * Bit of a misnomer, what this really means is - * to enable OS management of the system... aka - * to disable special hardware management features - */ -static void -lem_init_manageability(struct adapter *adapter) -{ - /* A shared code workaround */ - if (adapter->has_manage) { - int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); - /* disable hardware interception of ARP */ - manc &= ~(E1000_MANC_ARP_EN); - E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); - } -} - -/* - * Give control back to hardware management - * controller if there is one. - */ -static void -lem_release_manageability(struct adapter *adapter) -{ - if (adapter->has_manage) { - int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); - - /* re-enable hardware interception of ARP */ - manc |= E1000_MANC_ARP_EN; - E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); - } -} - -/* - * lem_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. - * For ASF and Pass Through versions of f/w this means - * that the driver is loaded. For AMT version type f/w - * this means that the network i/f is open. - */ -static void -lem_get_hw_control(struct adapter *adapter) -{ - u32 ctrl_ext; - - ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); - E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, - ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); - return; -} - -/* - * lem_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. - * For ASF and Pass Through versions of f/w this means that - * the driver is no longer loaded. For AMT versions of the - * f/w this means that the network i/f is closed. - */ -static void -lem_release_hw_control(struct adapter *adapter) -{ - u32 ctrl_ext; - - if (!adapter->has_manage) - return; - - ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); - E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, - ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); - return; -} - -static int -lem_is_valid_ether_addr(u8 *addr) -{ - char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; - - if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { - return (FALSE); - } - - return (TRUE); -} - -/* -** Parse the interface capabilities with regard -** to both system management and wake-on-lan for -** later use. -*/ -static void -lem_get_wakeup(device_t dev) -{ - struct adapter *adapter = device_get_softc(dev); - u16 eeprom_data = 0, device_id, apme_mask; - - adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw); - apme_mask = EM_EEPROM_APME; - - switch (adapter->hw.mac.type) { - case e1000_82542: - case e1000_82543: - break; - case e1000_82544: - e1000_read_nvm(&adapter->hw, - NVM_INIT_CONTROL2_REG, 1, &eeprom_data); - apme_mask = EM_82544_APME; - break; - case e1000_82546: - case e1000_82546_rev_3: - if (adapter->hw.bus.func == 1) { - e1000_read_nvm(&adapter->hw, - NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); - break; - } else - e1000_read_nvm(&adapter->hw, - NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); - break; - default: - e1000_read_nvm(&adapter->hw, - NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); - break; - } - if (eeprom_data & apme_mask) - adapter->wol = (E1000_WUFC_MAG | E1000_WUFC_MC); - /* - * We have the eeprom settings, now apply the special cases - * where the eeprom may be wrong or the board won't support - * wake on lan on a particular port - */ - device_id = pci_get_device(dev); - switch (device_id) { - case E1000_DEV_ID_82546GB_PCIE: - adapter->wol = 0; - break; - case E1000_DEV_ID_82546EB_FIBER: - case E1000_DEV_ID_82546GB_FIBER: - /* Wake events only supported on port A for dual fiber - * regardless of eeprom setting */ - if (E1000_READ_REG(&adapter->hw, E1000_STATUS) & - E1000_STATUS_FUNC_1) - adapter->wol = 0; - break; - case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3: - /* if quad port adapter, disable WoL on all but port A */ - if (global_quad_port_a != 0) - adapter->wol = 0; - /* Reset for multiple quad port adapters */ - if (++global_quad_port_a == 4) - global_quad_port_a = 0; - break; - } - return; -} - - -/* - * Enable PCI Wake On Lan capability - */ -static void -lem_enable_wakeup(device_t dev) -{ - struct adapter *adapter = device_get_softc(dev); - if_t ifp = adapter->ifp; - u32 pmc, ctrl, ctrl_ext, rctl; - u16 status; - - if ((pci_find_cap(dev, PCIY_PMG, &pmc) != 0)) - return; - - /* Advertise the wakeup capability */ - ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); - ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3); - E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); - E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); - - /* Keep the laser running on Fiber adapters */ - if (adapter->hw.phy.media_type == e1000_media_type_fiber || - adapter->hw.phy.media_type == e1000_media_type_internal_serdes) { - ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); - ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA; - E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext); - } - - /* - ** Determine type of Wakeup: note that wol - ** is set with all bits on by default. - */ - if ((if_getcapenable(ifp) & IFCAP_WOL_MAGIC) == 0) - adapter->wol &= ~E1000_WUFC_MAG; - - if ((if_getcapenable(ifp) & IFCAP_WOL_MCAST) == 0) - adapter->wol &= ~E1000_WUFC_MC; - else { - rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); - rctl |= E1000_RCTL_MPE; - E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); - } - - if (adapter->hw.mac.type == e1000_pchlan) { - if (lem_enable_phy_wakeup(adapter)) - return; - } else { - E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); - E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); - } - - - /* Request PME */ - status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2); - status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE); - if (if_getcapenable(ifp) & IFCAP_WOL) - status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; - pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2); - - return; -} - -/* -** WOL in the newer chipset interfaces (pchlan) -** require thing to be copied into the phy -*/ -static int -lem_enable_phy_wakeup(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - u32 mreg, ret = 0; - u16 preg; - - /* copy MAC RARs to PHY RARs */ - for (int i = 0; i < adapter->hw.mac.rar_entry_count; i++) { - mreg = E1000_READ_REG(hw, E1000_RAL(i)); - e1000_write_phy_reg(hw, BM_RAR_L(i), (u16)(mreg & 0xFFFF)); - e1000_write_phy_reg(hw, BM_RAR_M(i), - (u16)((mreg >> 16) & 0xFFFF)); - mreg = E1000_READ_REG(hw, E1000_RAH(i)); - e1000_write_phy_reg(hw, BM_RAR_H(i), (u16)(mreg & 0xFFFF)); - e1000_write_phy_reg(hw, BM_RAR_CTRL(i), - (u16)((mreg >> 16) & 0xFFFF)); - } - - /* copy MAC MTA to PHY MTA */ - for (int i = 0; i < adapter->hw.mac.mta_reg_count; i++) { - mreg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i); - e1000_write_phy_reg(hw, BM_MTA(i), (u16)(mreg & 0xFFFF)); - e1000_write_phy_reg(hw, BM_MTA(i) + 1, - (u16)((mreg >> 16) & 0xFFFF)); - } - - /* configure PHY Rx Control register */ - e1000_read_phy_reg(&adapter->hw, BM_RCTL, &preg); - mreg = E1000_READ_REG(hw, E1000_RCTL); - if (mreg & E1000_RCTL_UPE) - preg |= BM_RCTL_UPE; - if (mreg & E1000_RCTL_MPE) - preg |= BM_RCTL_MPE; - preg &= ~(BM_RCTL_MO_MASK); - if (mreg & E1000_RCTL_MO_3) - preg |= (((mreg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT) - << BM_RCTL_MO_SHIFT); - if (mreg & E1000_RCTL_BAM) - preg |= BM_RCTL_BAM; - if (mreg & E1000_RCTL_PMCF) - preg |= BM_RCTL_PMCF; - mreg = E1000_READ_REG(hw, E1000_CTRL); - if (mreg & E1000_CTRL_RFCE) - preg |= BM_RCTL_RFCE; - e1000_write_phy_reg(&adapter->hw, BM_RCTL, preg); - - /* enable PHY wakeup in MAC register */ - E1000_WRITE_REG(hw, E1000_WUC, - E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN); - E1000_WRITE_REG(hw, E1000_WUFC, adapter->wol); - - /* configure and enable PHY wakeup in PHY registers */ - e1000_write_phy_reg(&adapter->hw, BM_WUFC, adapter->wol); - e1000_write_phy_reg(&adapter->hw, BM_WUC, E1000_WUC_PME_EN); - - /* activate PHY wakeup */ - ret = hw->phy.ops.acquire(hw); - if (ret) { - printf("Could not acquire PHY\n"); - return ret; - } - e1000_write_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT, - (BM_WUC_ENABLE_PAGE << IGP_PAGE_SHIFT)); - ret = e1000_read_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, &preg); - if (ret) { - printf("Could not read PHY page 769\n"); - goto out; - } - preg |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT; - ret = e1000_write_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, preg); - if (ret) - printf("Could not set PHY Host Wakeup bit\n"); -out: - hw->phy.ops.release(hw); - - return ret; -} - -static void -lem_led_func(void *arg, int onoff) -{ - struct adapter *adapter = arg; - - EM_CORE_LOCK(adapter); - if (onoff) { - e1000_setup_led(&adapter->hw); - e1000_led_on(&adapter->hw); - } else { - e1000_led_off(&adapter->hw); - e1000_cleanup_led(&adapter->hw); - } - EM_CORE_UNLOCK(adapter); -} - -/********************************************************************* -* 82544 Coexistence issue workaround. -* There are 2 issues. -* 1. Transmit Hang issue. -* To detect this issue, following equation can be used... -* SIZE[3:0] + ADDR[2:0] = SUM[3:0]. -* If SUM[3:0] is in between 1 to 4, we will have this issue. -* -* 2. DAC issue. -* To detect this issue, following equation can be used... -* SIZE[3:0] + ADDR[2:0] = SUM[3:0]. -* If SUM[3:0] is in between 9 to c, we will have this issue. -* -* -* WORKAROUND: -* Make sure we do not have ending address -* as 1,2,3,4(Hang) or 9,a,b,c (DAC) -* -*************************************************************************/ -static u32 -lem_fill_descriptors (bus_addr_t address, u32 length, - PDESC_ARRAY desc_array) -{ - u32 safe_terminator; - - /* Since issue is sensitive to length and address.*/ - /* Let us first check the address...*/ - if (length <= 4) { - desc_array->descriptor[0].address = address; - desc_array->descriptor[0].length = length; - desc_array->elements = 1; - return (desc_array->elements); - } - safe_terminator = (u32)((((u32)address & 0x7) + - (length & 0xF)) & 0xF); - /* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */ - if (safe_terminator == 0 || - (safe_terminator > 4 && - safe_terminator < 9) || - (safe_terminator > 0xC && - safe_terminator <= 0xF)) { - desc_array->descriptor[0].address = address; - desc_array->descriptor[0].length = length; - desc_array->elements = 1; - return (desc_array->elements); - } - - desc_array->descriptor[0].address = address; - desc_array->descriptor[0].length = length - 4; - desc_array->descriptor[1].address = address + (length - 4); - desc_array->descriptor[1].length = 4; - desc_array->elements = 2; - return (desc_array->elements); -} - -/********************************************************************** - * - * Update the board statistics counters. - * - **********************************************************************/ -static void -lem_update_stats_counters(struct adapter *adapter) -{ - - if(adapter->hw.phy.media_type == e1000_media_type_copper || - (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) { - adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS); - adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC); - } - adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS); - adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC); - adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC); - adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL); - - adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC); - adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL); - adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC); - adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC); - adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC); - adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC); - adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC); - adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC); - adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC); - adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC); - adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64); - adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127); - adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255); - adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511); - adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023); - adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522); - adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC); - adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC); - adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC); - adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC); - - /* For the 64-bit byte counters the low dword must be read first. */ - /* Both registers clear on the read of the high dword */ - - adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCL) + - ((u64)E1000_READ_REG(&adapter->hw, E1000_GORCH) << 32); - adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCL) + - ((u64)E1000_READ_REG(&adapter->hw, E1000_GOTCH) << 32); - - adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC); - adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC); - adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC); - adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC); - adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC); - - adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH); - adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH); - - adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR); - adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT); - adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64); - adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127); - adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255); - adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511); - adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023); - adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522); - adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC); - adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC); - - if (adapter->hw.mac.type >= e1000_82543) { - adapter->stats.algnerrc += - E1000_READ_REG(&adapter->hw, E1000_ALGNERRC); - adapter->stats.rxerrc += - E1000_READ_REG(&adapter->hw, E1000_RXERRC); - adapter->stats.tncrs += - E1000_READ_REG(&adapter->hw, E1000_TNCRS); - adapter->stats.cexterr += - E1000_READ_REG(&adapter->hw, E1000_CEXTERR); - adapter->stats.tsctc += - E1000_READ_REG(&adapter->hw, E1000_TSCTC); - adapter->stats.tsctfc += - E1000_READ_REG(&adapter->hw, E1000_TSCTFC); - } -} - -static uint64_t -lem_get_counter(if_t ifp, ift_counter cnt) -{ - struct adapter *adapter; - - adapter = if_getsoftc(ifp); - - switch (cnt) { - case IFCOUNTER_COLLISIONS: - return (adapter->stats.colc); - case IFCOUNTER_IERRORS: - return (adapter->dropped_pkts + adapter->stats.rxerrc + - adapter->stats.crcerrs + adapter->stats.algnerrc + - adapter->stats.ruc + adapter->stats.roc + - adapter->stats.mpc + adapter->stats.cexterr); - case IFCOUNTER_OERRORS: - return (adapter->stats.ecol + adapter->stats.latecol + - adapter->watchdog_events); - default: - return (if_get_counter_default(ifp, cnt)); - } -} - -/* Export a single 32-bit register via a read-only sysctl. */ -static int -lem_sysctl_reg_handler(SYSCTL_HANDLER_ARGS) -{ - struct adapter *adapter; - u_int val; - - adapter = oidp->oid_arg1; - val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2); - return (sysctl_handle_int(oidp, &val, 0, req)); -} - -/* - * Add sysctl variables, one per statistic, to the system. - */ -static void -lem_add_hw_stats(struct adapter *adapter) -{ - device_t dev = adapter->dev; - - struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); - struct sysctl_oid *tree = device_get_sysctl_tree(dev); - struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); - struct e1000_hw_stats *stats = &adapter->stats; - - struct sysctl_oid *stat_node; - struct sysctl_oid_list *stat_list; - - /* Driver Statistics */ - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "cluster_alloc_fail", - CTLFLAG_RD, &adapter->mbuf_cluster_failed, - "Std mbuf cluster failed"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_fail", - CTLFLAG_RD, &adapter->mbuf_defrag_failed, - "Defragmenting mbuf chain failed"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", - CTLFLAG_RD, &adapter->dropped_pkts, - "Driver dropped packets"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail", - CTLFLAG_RD, &adapter->no_tx_dma_setup, - "Driver tx dma failure in xmit"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_desc_fail1", - CTLFLAG_RD, &adapter->no_tx_desc_avail1, - "Not enough tx descriptors failure in xmit"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_desc_fail2", - CTLFLAG_RD, &adapter->no_tx_desc_avail2, - "Not enough tx descriptors failure in xmit"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns", - CTLFLAG_RD, &adapter->rx_overruns, - "RX overruns"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts", - CTLFLAG_RD, &adapter->watchdog_events, - "Watchdog timeouts"); - - SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control", - CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_CTRL, - lem_sysctl_reg_handler, "IU", - "Device Control Register"); - SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_control", - CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RCTL, - lem_sysctl_reg_handler, "IU", - "Receiver Control Register"); - SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water", - CTLFLAG_RD, &adapter->hw.fc.high_water, 0, - "Flow Control High Watermark"); - SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", - CTLFLAG_RD, &adapter->hw.fc.low_water, 0, - "Flow Control Low Watermark"); - SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "fifo_workaround", - CTLFLAG_RD, &adapter->tx_fifo_wrk_cnt, - "TX FIFO workaround events"); - SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "fifo_reset", - CTLFLAG_RD, &adapter->tx_fifo_reset_cnt, - "TX FIFO resets"); - - SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "txd_head", - CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDH(0), - lem_sysctl_reg_handler, "IU", - "Transmit Descriptor Head"); - SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "txd_tail", - CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDT(0), - lem_sysctl_reg_handler, "IU", - "Transmit Descriptor Tail"); - SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rxd_head", - CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDH(0), - lem_sysctl_reg_handler, "IU", - "Receive Descriptor Head"); - SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rxd_tail", - CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDT(0), - lem_sysctl_reg_handler, "IU", - "Receive Descriptor Tail"); - - - /* MAC stats get their own sub node */ - - stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", - CTLFLAG_RD, NULL, "Statistics"); - stat_list = SYSCTL_CHILDREN(stat_node); - - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "excess_coll", - CTLFLAG_RD, &stats->ecol, - "Excessive collisions"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "single_coll", - CTLFLAG_RD, &stats->scc, - "Single collisions"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "multiple_coll", - CTLFLAG_RD, &stats->mcc, - "Multiple collisions"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "late_coll", - CTLFLAG_RD, &stats->latecol, - "Late collisions"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "collision_count", - CTLFLAG_RD, &stats->colc, - "Collision Count"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "symbol_errors", - CTLFLAG_RD, &adapter->stats.symerrs, - "Symbol Errors"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "sequence_errors", - CTLFLAG_RD, &adapter->stats.sec, - "Sequence Errors"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "defer_count", - CTLFLAG_RD, &adapter->stats.dc, - "Defer Count"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "missed_packets", - CTLFLAG_RD, &adapter->stats.mpc, - "Missed Packets"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_no_buff", - CTLFLAG_RD, &adapter->stats.rnbc, - "Receive No Buffers"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersize", - CTLFLAG_RD, &adapter->stats.ruc, - "Receive Undersize"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented", - CTLFLAG_RD, &adapter->stats.rfc, - "Fragmented Packets Received "); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversize", - CTLFLAG_RD, &adapter->stats.roc, - "Oversized Packets Received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabber", - CTLFLAG_RD, &adapter->stats.rjc, - "Recevied Jabber"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_errs", - CTLFLAG_RD, &adapter->stats.rxerrc, - "Receive Errors"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs", - CTLFLAG_RD, &adapter->stats.crcerrs, - "CRC errors"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "alignment_errs", - CTLFLAG_RD, &adapter->stats.algnerrc, - "Alignment Errors"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs", - CTLFLAG_RD, &adapter->stats.cexterr, - "Collision/Carrier extension errors"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd", - CTLFLAG_RD, &adapter->stats.xonrxc, - "XON Received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd", - CTLFLAG_RD, &adapter->stats.xontxc, - "XON Transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd", - CTLFLAG_RD, &adapter->stats.xoffrxc, - "XOFF Received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd", - CTLFLAG_RD, &adapter->stats.xofftxc, - "XOFF Transmitted"); - - /* Packet Reception Stats */ - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd", - CTLFLAG_RD, &adapter->stats.tpr, - "Total Packets Received "); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd", - CTLFLAG_RD, &adapter->stats.gprc, - "Good Packets Received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd", - CTLFLAG_RD, &adapter->stats.bprc, - "Broadcast Packets Received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd", - CTLFLAG_RD, &adapter->stats.mprc, - "Multicast Packets Received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64", - CTLFLAG_RD, &adapter->stats.prc64, - "64 byte frames received "); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127", - CTLFLAG_RD, &adapter->stats.prc127, - "65-127 byte frames received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255", - CTLFLAG_RD, &adapter->stats.prc255, - "128-255 byte frames received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511", - CTLFLAG_RD, &adapter->stats.prc511, - "256-511 byte frames received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023", - CTLFLAG_RD, &adapter->stats.prc1023, - "512-1023 byte frames received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522", - CTLFLAG_RD, &adapter->stats.prc1522, - "1023-1522 byte frames received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", - CTLFLAG_RD, &adapter->stats.gorc, - "Good Octets Received"); - - /* Packet Transmission Stats */ - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", - CTLFLAG_RD, &adapter->stats.gotc, - "Good Octets Transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd", - CTLFLAG_RD, &adapter->stats.tpt, - "Total Packets Transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", - CTLFLAG_RD, &adapter->stats.gptc, - "Good Packets Transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd", - CTLFLAG_RD, &adapter->stats.bptc, - "Broadcast Packets Transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd", - CTLFLAG_RD, &adapter->stats.mptc, - "Multicast Packets Transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64", - CTLFLAG_RD, &adapter->stats.ptc64, - "64 byte frames transmitted "); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127", - CTLFLAG_RD, &adapter->stats.ptc127, - "65-127 byte frames transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255", - CTLFLAG_RD, &adapter->stats.ptc255, - "128-255 byte frames transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511", - CTLFLAG_RD, &adapter->stats.ptc511, - "256-511 byte frames transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023", - CTLFLAG_RD, &adapter->stats.ptc1023, - "512-1023 byte frames transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522", - CTLFLAG_RD, &adapter->stats.ptc1522, - "1024-1522 byte frames transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_txd", - CTLFLAG_RD, &adapter->stats.tsctc, - "TSO Contexts Transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail", - CTLFLAG_RD, &adapter->stats.tsctfc, - "TSO Contexts Failed"); -} - -/********************************************************************** - * - * This routine provides a way to dump out the adapter eeprom, - * often a useful debug/service tool. This only dumps the first - * 32 words, stuff that matters is in that extent. - * - **********************************************************************/ - -static int -lem_sysctl_nvm_info(SYSCTL_HANDLER_ARGS) -{ - struct adapter *adapter; - int error; - int result; - - result = -1; - error = sysctl_handle_int(oidp, &result, 0, req); - - if (error || !req->newptr) - return (error); - - /* - * This value will cause a hex dump of the - * first 32 16-bit words of the EEPROM to - * the screen. - */ - if (result == 1) { - adapter = (struct adapter *)arg1; - lem_print_nvm_info(adapter); - } - - return (error); -} - -static void -lem_print_nvm_info(struct adapter *adapter) -{ - u16 eeprom_data; - int i, j, row = 0; - - /* Its a bit crude, but it gets the job done */ - printf("\nInterface EEPROM Dump:\n"); - printf("Offset\n0x0000 "); - for (i = 0, j = 0; i < 32; i++, j++) { - if (j == 8) { /* Make the offset block */ - j = 0; ++row; - printf("\n0x00%x0 ",row); - } - e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data); - printf("%04x ", eeprom_data); - } - printf("\n"); -} - -static int -lem_sysctl_int_delay(SYSCTL_HANDLER_ARGS) -{ - struct em_int_delay_info *info; - struct adapter *adapter; - u32 regval; - int error; - int usecs; - int ticks; - - info = (struct em_int_delay_info *)arg1; - usecs = info->value; - error = sysctl_handle_int(oidp, &usecs, 0, req); - if (error != 0 || req->newptr == NULL) - return (error); - if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535)) - return (EINVAL); - info->value = usecs; - ticks = EM_USECS_TO_TICKS(usecs); - if (info->offset == E1000_ITR) /* units are 256ns here */ - ticks *= 4; - - adapter = info->adapter; - - EM_CORE_LOCK(adapter); - regval = E1000_READ_OFFSET(&adapter->hw, info->offset); - regval = (regval & ~0xffff) | (ticks & 0xffff); - /* Handle a few special cases. */ - switch (info->offset) { - case E1000_RDTR: - break; - case E1000_TIDV: - if (ticks == 0) { - adapter->txd_cmd &= ~E1000_TXD_CMD_IDE; - /* Don't write 0 into the TIDV register. */ - regval++; - } else - adapter->txd_cmd |= E1000_TXD_CMD_IDE; - break; - } - E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval); - EM_CORE_UNLOCK(adapter); - return (0); -} - -static void -lem_add_int_delay_sysctl(struct adapter *adapter, const char *name, - const char *description, struct em_int_delay_info *info, - int offset, int value) -{ - info->adapter = adapter; - info->offset = offset; - info->value = value; - SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), - OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, - info, 0, lem_sysctl_int_delay, "I", description); -} - -static void -lem_set_flow_cntrl(struct adapter *adapter, const char *name, - const char *description, int *limit, int value) -{ - *limit = value; - SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), - OID_AUTO, name, CTLFLAG_RW, limit, value, description); -} - -static void -lem_add_rx_process_limit(struct adapter *adapter, const char *name, - const char *description, int *limit, int value) -{ - *limit = value; - SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), - OID_AUTO, name, CTLFLAG_RW, limit, value, description); -} diff --git a/freebsd/sys/dev/e1000/if_lem.h b/freebsd/sys/dev/e1000/if_lem.h deleted file mode 100644 index 4a27c34b..00000000 --- a/freebsd/sys/dev/e1000/if_lem.h +++ /dev/null @@ -1,519 +0,0 @@ -/****************************************************************************** - - Copyright (c) 2001-2015, Intel Corporation - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - -******************************************************************************/ -/*$FreeBSD$*/ - - -#ifndef _LEM_H_DEFINED_ -#define _LEM_H_DEFINED_ - - -/* Tunables */ - -/* - * EM_TXD: Maximum number of Transmit Descriptors - * Valid Range: 80-256 for 82542 and 82543-based adapters - * 80-4096 for others - * Default Value: 256 - * This value is the number of transmit descriptors allocated by the driver. - * Increasing this value allows the driver to queue more transmits. Each - * descriptor is 16 bytes. - * Since TDLEN should be multiple of 128bytes, the number of transmit - * desscriptors should meet the following condition. - * (num_tx_desc * sizeof(struct e1000_tx_desc)) % 128 == 0 - */ -#define EM_MIN_TXD 80 -#define EM_MAX_TXD_82543 256 -#define EM_MAX_TXD 4096 -#define EM_DEFAULT_TXD EM_MAX_TXD_82543 - -/* - * EM_RXD - Maximum number of receive Descriptors - * Valid Range: 80-256 for 82542 and 82543-based adapters - * 80-4096 for others - * Default Value: 256 - * This value is the number of receive descriptors allocated by the driver. - * Increasing this value allows the driver to buffer more incoming packets. - * Each descriptor is 16 bytes. A receive buffer is also allocated for each - * descriptor. The maximum MTU size is 16110. - * Since TDLEN should be multiple of 128bytes, the number of transmit - * desscriptors should meet the following condition. - * (num_tx_desc * sizeof(struct e1000_tx_desc)) % 128 == 0 - */ -#define EM_MIN_RXD 80 -#define EM_MAX_RXD_82543 256 -#define EM_MAX_RXD 4096 -#define EM_DEFAULT_RXD EM_MAX_RXD_82543 - -/* - * EM_TIDV - Transmit Interrupt Delay Value - * Valid Range: 0-65535 (0=off) - * Default Value: 64 - * This value delays the generation of transmit interrupts in units of - * 1.024 microseconds. Transmit interrupt reduction can improve CPU - * efficiency if properly tuned for specific network traffic. If the - * system is reporting dropped transmits, this value may be set too high - * causing the driver to run out of available transmit descriptors. - */ -#define EM_TIDV 64 - -/* - * EM_TADV - Transmit Absolute Interrupt Delay Value - * (Not valid for 82542/82543/82544) - * Valid Range: 0-65535 (0=off) - * Default Value: 64 - * This value, in units of 1.024 microseconds, limits the delay in which a - * transmit interrupt is generated. Useful only if EM_TIDV is non-zero, - * this value ensures that an interrupt is generated after the initial - * packet is sent on the wire within the set amount of time. Proper tuning, - * along with EM_TIDV, may improve traffic throughput in specific - * network conditions. - */ -#define EM_TADV 64 - -/* - * EM_RDTR - Receive Interrupt Delay Timer (Packet Timer) - * Valid Range: 0-65535 (0=off) - * Default Value: 0 - * This value delays the generation of receive interrupts in units of 1.024 - * microseconds. Receive interrupt reduction can improve CPU efficiency if - * properly tuned for specific network traffic. Increasing this value adds - * extra latency to frame reception and can end up decreasing the throughput - * of TCP traffic. If the system is reporting dropped receives, this value - * may be set too high, causing the driver to run out of available receive - * descriptors. - * - * CAUTION: When setting EM_RDTR to a value other than 0, adapters - * may hang (stop transmitting) under certain network conditions. - * If this occurs a WATCHDOG message is logged in the system - * event log. In addition, the controller is automatically reset, - * restoring the network connection. To eliminate the potential - * for the hang ensure that EM_RDTR is set to 0. - */ -#define EM_RDTR 0 - -/* - * Receive Interrupt Absolute Delay Timer (Not valid for 82542/82543/82544) - * Valid Range: 0-65535 (0=off) - * Default Value: 64 - * This value, in units of 1.024 microseconds, limits the delay in which a - * receive interrupt is generated. Useful only if EM_RDTR is non-zero, - * this value ensures that an interrupt is generated after the initial - * packet is received within the set amount of time. Proper tuning, - * along with EM_RDTR, may improve traffic throughput in specific network - * conditions. - */ -#define EM_RADV 64 - -/* - * This parameter controls the max duration of transmit watchdog. - */ -#define EM_WATCHDOG (10 * hz) - -/* - * This parameter controls when the driver calls the routine to reclaim - * transmit descriptors. - */ -#define EM_TX_CLEANUP_THRESHOLD (adapter->num_tx_desc / 8) -#define EM_TX_OP_THRESHOLD (adapter->num_tx_desc / 32) - -/* - * This parameter controls whether or not autonegotation is enabled. - * 0 - Disable autonegotiation - * 1 - Enable autonegotiation - */ -#define DO_AUTO_NEG 1 - -/* - * This parameter control whether or not the driver will wait for - * autonegotiation to complete. - * 1 - Wait for autonegotiation to complete - * 0 - Don't wait for autonegotiation to complete - */ -#define WAIT_FOR_AUTO_NEG_DEFAULT 0 - -/* Tunables -- End */ - -#define AUTONEG_ADV_DEFAULT (ADVERTISE_10_HALF | ADVERTISE_10_FULL | \ - ADVERTISE_100_HALF | ADVERTISE_100_FULL | \ - ADVERTISE_1000_FULL) - -#define AUTO_ALL_MODES 0 - -/* PHY master/slave setting */ -#define EM_MASTER_SLAVE e1000_ms_hw_default - -/* - * Micellaneous constants - */ -#define EM_VENDOR_ID 0x8086 -#define EM_FLASH 0x0014 - -#define EM_JUMBO_PBA 0x00000028 -#define EM_DEFAULT_PBA 0x00000030 -#define EM_SMARTSPEED_DOWNSHIFT 3 -#define EM_SMARTSPEED_MAX 15 -#define EM_MAX_LOOP 10 - -#define MAX_NUM_MULTICAST_ADDRESSES 128 -#define PCI_ANY_ID (~0U) -#define ETHER_ALIGN 2 -#define EM_FC_PAUSE_TIME 0x0680 -#define EM_EEPROM_APME 0x400; -#define EM_82544_APME 0x0004; - -/* Code compatilbility between 6 and 7 */ -#ifndef ETHER_BPF_MTAP -#define ETHER_BPF_MTAP BPF_MTAP -#endif - -/* - * TDBA/RDBA should be aligned on 16 byte boundary. But TDLEN/RDLEN should be - * multiple of 128 bytes. So we align TDBA/RDBA on 128 byte boundary. This will - * also optimize cache line size effect. H/W supports up to cache line size 128. - */ -#define EM_DBA_ALIGN 128 - -#define SPEED_MODE_BIT (1<<21) /* On PCI-E MACs only */ - -/* PCI Config defines */ -#define EM_BAR_TYPE(v) ((v) & EM_BAR_TYPE_MASK) -#define EM_BAR_TYPE_MASK 0x00000001 -#define EM_BAR_TYPE_MMEM 0x00000000 -#define EM_BAR_TYPE_IO 0x00000001 -#define EM_BAR_TYPE_FLASH 0x0014 -#define EM_BAR_MEM_TYPE(v) ((v) & EM_BAR_MEM_TYPE_MASK) -#define EM_BAR_MEM_TYPE_MASK 0x00000006 -#define EM_BAR_MEM_TYPE_32BIT 0x00000000 -#define EM_BAR_MEM_TYPE_64BIT 0x00000004 -#define EM_MSIX_BAR 3 /* On 82575 */ - -#if __FreeBSD_version < 900000 -#define SYSCTL_ADD_UQUAD SYSCTL_ADD_QUAD -#endif - -/* Defines for printing debug information */ -#define DEBUG_INIT 0 -#define DEBUG_IOCTL 0 -#define DEBUG_HW 0 - -#define INIT_DEBUGOUT(S) if (DEBUG_INIT) printf(S "\n") -#define INIT_DEBUGOUT1(S, A) if (DEBUG_INIT) printf(S "\n", A) -#define INIT_DEBUGOUT2(S, A, B) if (DEBUG_INIT) printf(S "\n", A, B) -#define IOCTL_DEBUGOUT(S) if (DEBUG_IOCTL) printf(S "\n") -#define IOCTL_DEBUGOUT1(S, A) if (DEBUG_IOCTL) printf(S "\n", A) -#define IOCTL_DEBUGOUT2(S, A, B) if (DEBUG_IOCTL) printf(S "\n", A, B) -#define HW_DEBUGOUT(S) if (DEBUG_HW) printf(S "\n") -#define HW_DEBUGOUT1(S, A) if (DEBUG_HW) printf(S "\n", A) -#define HW_DEBUGOUT2(S, A, B) if (DEBUG_HW) printf(S "\n", A, B) - -#define EM_MAX_SCATTER 40 -#define EM_VFTA_SIZE 128 -#define EM_MSIX_MASK 0x01F00000 /* For 82574 use */ -#define ETH_ZLEN 60 -#define ETH_ADDR_LEN 6 -#define CSUM_OFFLOAD 7 /* Offload bits in mbuf flag */ - -/* - * 82574 has a nonstandard address for EIAC - * and since its only used in MSIX, and in - * the em driver only 82574 uses MSIX we can - * solve it just using this define. - */ -#define EM_EIAC 0x000DC - -/* Used in for 82547 10Mb Half workaround */ -#define EM_PBA_BYTES_SHIFT 0xA -#define EM_TX_HEAD_ADDR_SHIFT 7 -#define EM_PBA_TX_MASK 0xFFFF0000 -#define EM_FIFO_HDR 0x10 -#define EM_82547_PKT_THRESH 0x3e0 - -/* Precision Time Sync (IEEE 1588) defines */ -#define ETHERTYPE_IEEE1588 0x88F7 -#define PICOSECS_PER_TICK 20833 -#define TSYNC_PORT 319 /* UDP port for the protocol */ - -#ifdef NIC_PARAVIRT -#define E1000_PARA_SUBDEV 0x1101 /* special id */ -#define E1000_CSBAL 0x02830 /* csb phys. addr. low */ -#define E1000_CSBAH 0x02834 /* csb phys. addr. hi */ -#include <net/paravirt.h> -#endif /* NIC_PARAVIRT */ - -/* - * Bus dma allocation structure used by - * e1000_dma_malloc and e1000_dma_free. - */ -struct em_dma_alloc { - bus_addr_t dma_paddr; - caddr_t dma_vaddr; - bus_dma_tag_t dma_tag; - bus_dmamap_t dma_map; - bus_dma_segment_t dma_seg; - int dma_nseg; -}; - -struct adapter; - -struct em_int_delay_info { - struct adapter *adapter; /* Back-pointer to the adapter struct */ - int offset; /* Register offset to read/write */ - int value; /* Current value in usecs */ -}; - -/* Our adapter structure */ -struct adapter { - if_t ifp; - struct e1000_hw hw; - - /* FreeBSD operating-system-specific structures. */ - struct e1000_osdep osdep; - device_t dev; - struct cdev *led_dev; - - struct resource *memory; - struct resource *flash; - struct resource *msix; - - struct resource *ioport; - int io_rid; - - /* 82574 may use 3 int vectors */ - struct resource *res[3]; - void *tag[3]; - int rid[3]; - - struct ifmedia media; - struct callout timer; - struct callout tx_fifo_timer; - bool watchdog_check; - int watchdog_time; - int msi; - int if_flags; - int max_frame_size; - int min_frame_size; - struct mtx core_mtx; - struct mtx tx_mtx; - struct mtx rx_mtx; - int em_insert_vlan_header; - - /* Task for FAST handling */ - struct task link_task; - struct task rxtx_task; - struct task rx_task; - struct task tx_task; - struct taskqueue *tq; /* private task queue */ - - eventhandler_tag vlan_attach; - eventhandler_tag vlan_detach; - u32 num_vlans; - - /* Management and WOL features */ - u32 wol; - bool has_manage; - bool has_amt; - - /* Multicast array memory */ - u8 *mta; - - /* - ** Shadow VFTA table, this is needed because - ** the real vlan filter table gets cleared during - ** a soft reset and the driver needs to be able - ** to repopulate it. - */ - u32 shadow_vfta[EM_VFTA_SIZE]; - - /* Info about the interface */ - uint8_t link_active; - uint16_t link_speed; - uint16_t link_duplex; - uint32_t smartspeed; - uint32_t fc_setting; - - struct em_int_delay_info tx_int_delay; - struct em_int_delay_info tx_abs_int_delay; - struct em_int_delay_info rx_int_delay; - struct em_int_delay_info rx_abs_int_delay; - struct em_int_delay_info tx_itr; - - /* - * Transmit definitions - * - * We have an array of num_tx_desc descriptors (handled - * by the controller) paired with an array of tx_buffers - * (at tx_buffer_area). - * The index of the next available descriptor is next_avail_tx_desc. - * The number of remaining tx_desc is num_tx_desc_avail. - */ - struct em_dma_alloc txdma; /* bus_dma glue for tx desc */ - struct e1000_tx_desc *tx_desc_base; - uint32_t next_avail_tx_desc; - uint32_t next_tx_to_clean; - volatile uint16_t num_tx_desc_avail; - uint16_t num_tx_desc; - uint16_t last_hw_offload; - uint32_t txd_cmd; - struct em_buffer *tx_buffer_area; - bus_dma_tag_t txtag; /* dma tag for tx */ - uint32_t tx_tso; /* last tx was tso */ - - /* - * Receive definitions - * - * we have an array of num_rx_desc rx_desc (handled by the - * controller), and paired with an array of rx_buffers - * (at rx_buffer_area). - * The next pair to check on receive is at offset next_rx_desc_to_check - */ - struct em_dma_alloc rxdma; /* bus_dma glue for rx desc */ - struct e1000_rx_desc *rx_desc_base; - uint32_t next_rx_desc_to_check; - uint32_t rx_buffer_len; - uint16_t num_rx_desc; - int rx_process_limit; - struct em_buffer *rx_buffer_area; - bus_dma_tag_t rxtag; - bus_dmamap_t rx_sparemap; - - /* - * First/last mbuf pointers, for - * collecting multisegment RX packets. - */ - struct mbuf *fmp; - struct mbuf *lmp; - - /* Misc stats maintained by the driver */ - unsigned long dropped_pkts; - unsigned long link_irq; - unsigned long mbuf_cluster_failed; - unsigned long mbuf_defrag_failed; - unsigned long no_tx_desc_avail1; - unsigned long no_tx_desc_avail2; - unsigned long no_tx_dma_setup; - unsigned long no_tx_map_avail; - unsigned long watchdog_events; - unsigned long rx_irq; - unsigned long rx_overruns; - unsigned long tx_irq; - - /* 82547 workaround */ - uint32_t tx_fifo_size; - uint32_t tx_fifo_head; - uint32_t tx_fifo_head_addr; - uint64_t tx_fifo_reset_cnt; - uint64_t tx_fifo_wrk_cnt; - uint32_t tx_head_addr; - - /* For 82544 PCIX Workaround */ - boolean_t pcix_82544; - boolean_t in_detach; - -#ifdef NIC_SEND_COMBINING - /* 0 = idle; 1xxxx int-pending; 3xxxx int + d pending + tdt */ -#define MIT_PENDING_INT 0x10000 /* pending interrupt */ -#define MIT_PENDING_TDT 0x30000 /* both intr and tdt write are pending */ - uint32_t shadow_tdt; - uint32_t sc_enable; -#endif /* NIC_SEND_COMBINING */ -#ifdef BATCH_DISPATCH - uint32_t batch_enable; -#endif /* BATCH_DISPATCH */ - -#ifdef NIC_PARAVIRT - struct em_dma_alloc csb_mem; /* phys address */ - struct paravirt_csb *csb; /* virtual addr */ - uint32_t rx_retries; /* optimize rx loop */ - uint32_t tdt_csb_count;// XXX stat - uint32_t tdt_reg_count;// XXX stat - uint32_t tdt_int_count;// XXX stat - uint32_t guest_need_kick_count;// XXX stat -#endif /* NIC_PARAVIRT */ - - struct e1000_hw_stats stats; -}; - -/* ****************************************************************************** - * vendor_info_array - * - * This array contains the list of Subvendor/Subdevice IDs on which the driver - * should load. - * - * ******************************************************************************/ -typedef struct _em_vendor_info_t { - unsigned int vendor_id; - unsigned int device_id; - unsigned int subvendor_id; - unsigned int subdevice_id; - unsigned int index; -} em_vendor_info_t; - -struct em_buffer { - int next_eop; /* Index of the desc to watch */ - struct mbuf *m_head; - bus_dmamap_t map; /* bus_dma map for packet */ -}; - -/* For 82544 PCIX Workaround */ -typedef struct _ADDRESS_LENGTH_PAIR -{ - uint64_t address; - uint32_t length; -} ADDRESS_LENGTH_PAIR, *PADDRESS_LENGTH_PAIR; - -typedef struct _DESCRIPTOR_PAIR -{ - ADDRESS_LENGTH_PAIR descriptor[4]; - uint32_t elements; -} DESC_ARRAY, *PDESC_ARRAY; - -#define EM_CORE_LOCK_INIT(_sc, _name) \ - mtx_init(&(_sc)->core_mtx, _name, "EM Core Lock", MTX_DEF) -#define EM_TX_LOCK_INIT(_sc, _name) \ - mtx_init(&(_sc)->tx_mtx, _name, "EM TX Lock", MTX_DEF) -#define EM_RX_LOCK_INIT(_sc, _name) \ - mtx_init(&(_sc)->rx_mtx, _name, "EM RX Lock", MTX_DEF) -#define EM_CORE_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->core_mtx) -#define EM_TX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->tx_mtx) -#define EM_RX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->rx_mtx) -#define EM_CORE_LOCK(_sc) mtx_lock(&(_sc)->core_mtx) -#define EM_TX_LOCK(_sc) mtx_lock(&(_sc)->tx_mtx) -#define EM_TX_TRYLOCK(_sc) mtx_trylock(&(_sc)->tx_mtx) -#define EM_RX_LOCK(_sc) mtx_lock(&(_sc)->rx_mtx) -#define EM_CORE_UNLOCK(_sc) mtx_unlock(&(_sc)->core_mtx) -#define EM_TX_UNLOCK(_sc) mtx_unlock(&(_sc)->tx_mtx) -#define EM_RX_UNLOCK(_sc) mtx_unlock(&(_sc)->rx_mtx) -#define EM_CORE_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->core_mtx, MA_OWNED) -#define EM_TX_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->tx_mtx, MA_OWNED) - -#endif /* _LEM_H_DEFINED_ */ diff --git a/freebsd/sys/dev/e1000/igb_txrx.c b/freebsd/sys/dev/e1000/igb_txrx.c new file mode 100644 index 00000000..2ed24e2d --- /dev/null +++ b/freebsd/sys/dev/e1000/igb_txrx.c @@ -0,0 +1,586 @@ +#include <machine/rtems-bsd-kernel-space.h> + +/*- + * Copyright (c) 2016 Matt Macy <mmacy@nextbsd.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* $FreeBSD$ */ +#include "if_em.h" + +#ifdef RSS +#include <net/rss_config.h> +#include <netinet/in_rss.h> +#endif + +#ifdef VERBOSE_DEBUG +#define DPRINTF device_printf +#else +#define DPRINTF(...) +#endif + +/********************************************************************* + * Local Function prototypes + *********************************************************************/ +static int igb_isc_txd_encap(void *arg, if_pkt_info_t pi); +static void igb_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx); +static int igb_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear); + +static void igb_isc_rxd_refill(void *arg, if_rxd_update_t iru); + +static void igb_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx); +static int igb_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget); + +static int igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); + +static int igb_tx_ctx_setup(struct tx_ring *txr, if_pkt_info_t pi, u32 *cmd_type_len, u32 *olinfo_status); +static int igb_tso_setup(struct tx_ring *txr, if_pkt_info_t pi, u32 *cmd_type_len, u32 *olinfo_status); + +static void igb_rx_checksum(u32 staterr, if_rxd_info_t ri, u32 ptype); +static int igb_determine_rsstype(u16 pkt_info); + +extern void igb_if_enable_intr(if_ctx_t ctx); +extern int em_intr(void *arg); + +struct if_txrx igb_txrx = { + igb_isc_txd_encap, + igb_isc_txd_flush, + igb_isc_txd_credits_update, + igb_isc_rxd_available, + igb_isc_rxd_pkt_get, + igb_isc_rxd_refill, + igb_isc_rxd_flush, + em_intr +}; + +extern if_shared_ctx_t em_sctx; + +/********************************************************************** + * + * Setup work for hardware segmentation offload (TSO) on + * adapters using advanced tx descriptors + * + **********************************************************************/ +static int +igb_tso_setup(struct tx_ring *txr, if_pkt_info_t pi, u32 *cmd_type_len, u32 *olinfo_status) +{ + struct e1000_adv_tx_context_desc *TXD; + struct adapter *adapter = txr->adapter; + u32 type_tucmd_mlhl = 0, vlan_macip_lens = 0; + u32 mss_l4len_idx = 0; + u32 paylen; + + switch(pi->ipi_etype) { + case ETHERTYPE_IPV6: + type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6; + break; + case ETHERTYPE_IP: + type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; + /* Tell transmit desc to also do IPv4 checksum. */ + *olinfo_status |= E1000_TXD_POPTS_IXSM << 8; + break; + default: + panic("%s: CSUM_TSO but no supported IP version (0x%04x)", + __func__, ntohs(pi->ipi_etype)); + break; + } + + TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[pi->ipi_pidx]; + + /* This is used in the transmit desc in encap */ + paylen = pi->ipi_len - pi->ipi_ehdrlen - pi->ipi_ip_hlen - pi->ipi_tcp_hlen; + + /* VLAN MACLEN IPLEN */ + if (pi->ipi_mflags & M_VLANTAG) { + vlan_macip_lens |= (pi->ipi_vtag << E1000_ADVTXD_VLAN_SHIFT); + } + + vlan_macip_lens |= pi->ipi_ehdrlen << E1000_ADVTXD_MACLEN_SHIFT; + vlan_macip_lens |= pi->ipi_ip_hlen; + TXD->vlan_macip_lens = htole32(vlan_macip_lens); + + /* ADV DTYPE TUCMD */ + type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; + type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; + TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); + + /* MSS L4LEN IDX */ + mss_l4len_idx |= (pi->ipi_tso_segsz << E1000_ADVTXD_MSS_SHIFT); + mss_l4len_idx |= (pi->ipi_tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT); + /* 82575 needs the queue index added */ + if (adapter->hw.mac.type == e1000_82575) + mss_l4len_idx |= txr->me << 4; + TXD->mss_l4len_idx = htole32(mss_l4len_idx); + + TXD->seqnum_seed = htole32(0); + *cmd_type_len |= E1000_ADVTXD_DCMD_TSE; + *olinfo_status |= E1000_TXD_POPTS_TXSM << 8; + *olinfo_status |= paylen << E1000_ADVTXD_PAYLEN_SHIFT; + + return (1); +} + +/********************************************************************* + * + * Advanced Context Descriptor setup for VLAN, CSUM or TSO + * + **********************************************************************/ +static int +igb_tx_ctx_setup(struct tx_ring *txr, if_pkt_info_t pi, u32 *cmd_type_len, u32 *olinfo_status) +{ + struct e1000_adv_tx_context_desc *TXD; + struct adapter *adapter = txr->adapter; + u32 vlan_macip_lens, type_tucmd_mlhl; + u32 mss_l4len_idx; + mss_l4len_idx = vlan_macip_lens = type_tucmd_mlhl = 0; + int offload = TRUE; + + /* First check if TSO is to be used */ + if (pi->ipi_csum_flags & CSUM_TSO) + return (igb_tso_setup(txr, pi, cmd_type_len, olinfo_status)); + + /* Indicate the whole packet as payload when not doing TSO */ + *olinfo_status |= pi->ipi_len << E1000_ADVTXD_PAYLEN_SHIFT; + + /* Now ready a context descriptor */ + TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[pi->ipi_pidx]; + + /* + ** In advanced descriptors the vlan tag must + ** be placed into the context descriptor. Hence + ** we need to make one even if not doing offloads. + */ + if (pi->ipi_mflags & M_VLANTAG) { + vlan_macip_lens |= (pi->ipi_vtag << E1000_ADVTXD_VLAN_SHIFT); + } else if ((pi->ipi_csum_flags & IGB_CSUM_OFFLOAD) == 0) { + return (0); + } + + /* Set the ether header length */ + vlan_macip_lens |= pi->ipi_ehdrlen << E1000_ADVTXD_MACLEN_SHIFT; + + switch(pi->ipi_etype) { + case ETHERTYPE_IP: + type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; + break; + case ETHERTYPE_IPV6: + type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6; + break; + default: + offload = FALSE; + break; + } + + vlan_macip_lens |= pi->ipi_ip_hlen; + type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; + + switch (pi->ipi_ipproto) { + case IPPROTO_TCP: + if (pi->ipi_csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP)) + type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; + break; + case IPPROTO_UDP: + if (pi->ipi_csum_flags & (CSUM_IP_UDP | CSUM_IP6_UDP)) + type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP; + break; + case IPPROTO_SCTP: + if (pi->ipi_csum_flags & (CSUM_IP_SCTP | CSUM_IP6_SCTP)) + type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP; + break; + default: + offload = FALSE; + break; + } + + if (offload) /* For the TX descriptor setup */ + *olinfo_status |= E1000_TXD_POPTS_TXSM << 8; + + /* 82575 needs the queue index added */ + if (adapter->hw.mac.type == e1000_82575) + mss_l4len_idx = txr->me << 4; + + /* Now copy bits into descriptor */ + TXD->vlan_macip_lens = htole32(vlan_macip_lens); + TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); + TXD->seqnum_seed = htole32(0); + TXD->mss_l4len_idx = htole32(mss_l4len_idx); + + return (1); +} + +static int +igb_isc_txd_encap(void *arg, if_pkt_info_t pi) +{ + struct adapter *sc = arg; + if_softc_ctx_t scctx = sc->shared; + struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; + struct tx_ring *txr = &que->txr; + int nsegs = pi->ipi_nsegs; + bus_dma_segment_t *segs = pi->ipi_segs; + union e1000_adv_tx_desc *txd = NULL; + int i, j, first, pidx_last; + u32 olinfo_status, cmd_type_len, txd_flags; + qidx_t ntxd; + + pidx_last = olinfo_status = 0; + /* Basic descriptor defines */ + cmd_type_len = (E1000_ADVTXD_DTYP_DATA | + E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT); + + if (pi->ipi_mflags & M_VLANTAG) + cmd_type_len |= E1000_ADVTXD_DCMD_VLE; + + first = i = pi->ipi_pidx; + ntxd = scctx->isc_ntxd[0]; + txd_flags = pi->ipi_flags & IPI_TX_INTR ? E1000_ADVTXD_DCMD_RS : 0; + /* Consume the first descriptor */ + i += igb_tx_ctx_setup(txr, pi, &cmd_type_len, &olinfo_status); + if (i == scctx->isc_ntxd[0]) + i = 0; + + /* 82575 needs the queue index added */ + if (sc->hw.mac.type == e1000_82575) + olinfo_status |= txr->me << 4; + + for (j = 0; j < nsegs; j++) { + bus_size_t seglen; + bus_addr_t segaddr; + + txd = (union e1000_adv_tx_desc *)&txr->tx_base[i]; + seglen = segs[j].ds_len; + segaddr = htole64(segs[j].ds_addr); + + txd->read.buffer_addr = segaddr; + txd->read.cmd_type_len = htole32(E1000_TXD_CMD_IFCS | + cmd_type_len | seglen); + txd->read.olinfo_status = htole32(olinfo_status); + pidx_last = i; + if (++i == scctx->isc_ntxd[0]) { + i = 0; + } + } + if (txd_flags) { + txr->tx_rsq[txr->tx_rs_pidx] = pidx_last; + txr->tx_rs_pidx = (txr->tx_rs_pidx+1) & (ntxd-1); + MPASS(txr->tx_rs_pidx != txr->tx_rs_cidx); + } + + txd->read.cmd_type_len |= htole32(E1000_TXD_CMD_EOP | txd_flags); + pi->ipi_new_pidx = i; + + return (0); +} + +static void +igb_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx) +{ + struct adapter *adapter = arg; + struct em_tx_queue *que = &adapter->tx_queues[txqid]; + struct tx_ring *txr = &que->txr; + + E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), pidx); +} + +static int +igb_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear) +{ + struct adapter *adapter = arg; + if_softc_ctx_t scctx = adapter->shared; + struct em_tx_queue *que = &adapter->tx_queues[txqid]; + struct tx_ring *txr = &que->txr; + + qidx_t processed = 0; + int updated; + qidx_t cur, prev, ntxd, rs_cidx; + int32_t delta; + uint8_t status; + + rs_cidx = txr->tx_rs_cidx; + if (rs_cidx == txr->tx_rs_pidx) + return (0); + cur = txr->tx_rsq[rs_cidx]; + status = ((union e1000_adv_tx_desc *)&txr->tx_base[cur])->wb.status; + updated = !!(status & E1000_TXD_STAT_DD); + + if (!clear || !updated) + return (updated); + + prev = txr->tx_cidx_processed; + ntxd = scctx->isc_ntxd[0]; + do { + delta = (int32_t)cur - (int32_t)prev; + MPASS(prev == 0 || delta != 0); + if (delta < 0) + delta += ntxd; + + processed += delta; + prev = cur; + rs_cidx = (rs_cidx + 1) & (ntxd-1); + if (rs_cidx == txr->tx_rs_pidx) + break; + cur = txr->tx_rsq[rs_cidx]; + status = ((union e1000_adv_tx_desc *)&txr->tx_base[cur])->wb.status; + } while ((status & E1000_TXD_STAT_DD)); + + txr->tx_rs_cidx = rs_cidx; + txr->tx_cidx_processed = prev; + return (processed); +} + +static void +igb_isc_rxd_refill(void *arg, if_rxd_update_t iru) +{ + struct adapter *sc = arg; + if_softc_ctx_t scctx = sc->shared; + uint16_t rxqid = iru->iru_qsidx; + struct em_rx_queue *que = &sc->rx_queues[rxqid]; + union e1000_adv_rx_desc *rxd; + struct rx_ring *rxr = &que->rxr; + uint64_t *paddrs; + uint32_t next_pidx, pidx; + uint16_t count; + int i; + + paddrs = iru->iru_paddrs; + pidx = iru->iru_pidx; + count = iru->iru_count; + + for (i = 0, next_pidx = pidx; i < count; i++) { + rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[next_pidx]; + + rxd->read.pkt_addr = htole64(paddrs[i]); + if (++next_pidx == scctx->isc_nrxd[0]) + next_pidx = 0; + } +} + +static void +igb_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx) +{ + struct adapter *sc = arg; + struct em_rx_queue *que = &sc->rx_queues[rxqid]; + struct rx_ring *rxr = &que->rxr; + + E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx); +} + +static int +igb_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) +{ + struct adapter *sc = arg; + if_softc_ctx_t scctx = sc->shared; + struct em_rx_queue *que = &sc->rx_queues[rxqid]; + struct rx_ring *rxr = &que->rxr; + union e1000_adv_rx_desc *rxd; + u32 staterr = 0; + int cnt, i, iter; + + if (budget == 1) { + rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[idx]; + staterr = le32toh(rxd->wb.upper.status_error); + return (staterr & E1000_RXD_STAT_DD); + } + + for (iter = cnt = 0, i = idx; iter < scctx->isc_nrxd[0] && iter <= budget;) { + rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[i]; + staterr = le32toh(rxd->wb.upper.status_error); + + if ((staterr & E1000_RXD_STAT_DD) == 0) + break; + + if (++i == scctx->isc_nrxd[0]) { + i = 0; + } + + if (staterr & E1000_RXD_STAT_EOP) + cnt++; + iter++; + } + return (cnt); +} + +/**************************************************************** + * Routine sends data which has been dma'ed into host memory + * to upper layer. Initialize ri structure. + * + * Returns 0 upon success, errno on failure + ***************************************************************/ + +static int +igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) +{ + struct adapter *adapter = arg; + if_softc_ctx_t scctx = adapter->shared; + struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; + struct rx_ring *rxr = &que->rxr; + struct ifnet *ifp = iflib_get_ifp(adapter->ctx); + union e1000_adv_rx_desc *rxd; + + u16 pkt_info, len; + u16 vtag = 0; + u32 ptype; + u32 staterr = 0; + bool eop; + int i = 0; + int cidx = ri->iri_cidx; + + do { + rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[cidx]; + staterr = le32toh(rxd->wb.upper.status_error); + pkt_info = le16toh(rxd->wb.lower.lo_dword.hs_rss.pkt_info); + + MPASS ((staterr & E1000_RXD_STAT_DD) != 0); + + len = le16toh(rxd->wb.upper.length); + ptype = le32toh(rxd->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK; + + ri->iri_len += len; + rxr->rx_bytes += ri->iri_len; + + rxd->wb.upper.status_error = 0; + eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP); + + if (((adapter->hw.mac.type == e1000_i350) || + (adapter->hw.mac.type == e1000_i354)) && + (staterr & E1000_RXDEXT_STATERR_LB)) + vtag = be16toh(rxd->wb.upper.vlan); + else + vtag = le16toh(rxd->wb.upper.vlan); + + /* Make sure bad packets are discarded */ + if (eop && ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0)) { + adapter->dropped_pkts++; + ++rxr->rx_discarded; + return (EBADMSG); + } + ri->iri_frags[i].irf_flid = 0; + ri->iri_frags[i].irf_idx = cidx; + ri->iri_frags[i].irf_len = len; + + if (++cidx == scctx->isc_nrxd[0]) + cidx = 0; +#ifdef notyet + if (rxr->hdr_split == TRUE) { + ri->iri_frags[i].irf_flid = 1; + ri->iri_frags[i].irf_idx = cidx; + if (++cidx == scctx->isc_nrxd[0]) + cidx = 0; + } +#endif + i++; + } while (!eop); + + rxr->rx_packets++; + + if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) + igb_rx_checksum(staterr, ri, ptype); + + if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && + (staterr & E1000_RXD_STAT_VP) != 0) { + ri->iri_vtag = vtag; + ri->iri_flags |= M_VLANTAG; + } + ri->iri_flowid = + le32toh(rxd->wb.lower.hi_dword.rss); + ri->iri_rsstype = igb_determine_rsstype(pkt_info); + ri->iri_nfrags = i; + + return (0); +} + +/********************************************************************* + * + * Verify that the hardware indicated that the checksum is valid. + * Inform the stack about the status of checksum so that stack + * doesn't spend time verifying the checksum. + * + *********************************************************************/ +static void +igb_rx_checksum(u32 staterr, if_rxd_info_t ri, u32 ptype) +{ + u16 status = (u16)staterr; + u8 errors = (u8) (staterr >> 24); + bool sctp = FALSE; + + /* Ignore Checksum bit is set */ + if (status & E1000_RXD_STAT_IXSM) { + ri->iri_csum_flags = 0; + return; + } + + if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 && + (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0) + sctp = 1; + else + sctp = 0; + + if (status & E1000_RXD_STAT_IPCS) { + /* Did it pass? */ + if (!(errors & E1000_RXD_ERR_IPE)) { + /* IP Checksum Good */ + ri->iri_csum_flags = CSUM_IP_CHECKED; + ri->iri_csum_flags |= CSUM_IP_VALID; + } else + ri->iri_csum_flags = 0; + } + + if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) { + u64 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); + if (sctp) /* reassign */ + type = CSUM_SCTP_VALID; + /* Did it pass? */ + if (!(errors & E1000_RXD_ERR_TCPE)) { + ri->iri_csum_flags |= type; + if (sctp == 0) + ri->iri_csum_data = htons(0xffff); + } + } + return; +} + +/******************************************************************** + * + * Parse the packet type to determine the appropriate hash + * + ******************************************************************/ +static int +igb_determine_rsstype(u16 pkt_info) +{ + switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) { + case E1000_RXDADV_RSSTYPE_IPV4_TCP: + return M_HASHTYPE_RSS_TCP_IPV4; + case E1000_RXDADV_RSSTYPE_IPV4: + return M_HASHTYPE_RSS_IPV4; + case E1000_RXDADV_RSSTYPE_IPV6_TCP: + return M_HASHTYPE_RSS_TCP_IPV6; + case E1000_RXDADV_RSSTYPE_IPV6_EX: + return M_HASHTYPE_RSS_IPV6_EX; + case E1000_RXDADV_RSSTYPE_IPV6: + return M_HASHTYPE_RSS_IPV6; + case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX: + return M_HASHTYPE_RSS_TCP_IPV6_EX; + default: + return M_HASHTYPE_OPAQUE; + } +} diff --git a/freebsd/sys/dev/fdt/fdt_common.h b/freebsd/sys/dev/fdt/fdt_common.h index 81ce4bfa..904d3e18 100644 --- a/freebsd/sys/dev/fdt/fdt_common.h +++ b/freebsd/sys/dev/fdt/fdt_common.h @@ -71,12 +71,6 @@ extern vm_paddr_t fdt_immr_pa; extern vm_offset_t fdt_immr_va; extern vm_offset_t fdt_immr_size; -struct fdt_pm_mask_entry { - char *compat; - uint32_t mask; -}; -extern struct fdt_pm_mask_entry fdt_pm_mask_table[]; - #if defined(FDT_DTB_STATIC) extern u_char fdt_static_dtb; #endif diff --git a/freebsd/sys/dev/fdt/simplebus.c b/freebsd/sys/dev/fdt/simplebus.c index d981d065..fb099965 100644 --- a/freebsd/sys/dev/fdt/simplebus.c +++ b/freebsd/sys/dev/fdt/simplebus.c @@ -127,7 +127,7 @@ simplebus_probe(device_t dev) /* * FDT data puts a "simple-bus" compatible string on many things that - * have children but aren't really busses in our world. Without a + * have children but aren't really buses in our world. Without a * ranges property we will fail to attach, so just fail to probe too. */ if (!(ofw_bus_is_compatible(dev, "simple-bus") && diff --git a/freebsd/sys/dev/mmc/bridge.h b/freebsd/sys/dev/mmc/bridge.h index a26c31ec..a780ffae 100644 --- a/freebsd/sys/dev/mmc/bridge.h +++ b/freebsd/sys/dev/mmc/bridge.h @@ -52,7 +52,7 @@ */ #ifndef DEV_MMC_BRIDGE_H -#define DEV_MMC_BRIDGE_H +#define DEV_MMC_BRIDGE_H #include <sys/bus.h> @@ -60,7 +60,7 @@ * This file defines interfaces for the mmc bridge. The names chosen * are similar to or the same as the names used in Linux to allow for * easy porting of what Linux calls mmc host drivers. I use the - * FreeBSD terminology of bridge and bus for consistancy with other + * FreeBSD terminology of bridge and bus for consistency with other * drivers in the system. This file corresponds roughly to the Linux * linux/mmc/host.h file. * @@ -73,10 +73,9 @@ * to be added to the mmcbus file). * * Attached to the mmc bridge is an mmcbus. The mmcbus is described - * in dev/mmc/bus.h. + * in dev/mmc/mmcbus_if.m. */ - /* * mmc_ios is a structure that is used to store the state of the mmc/sd * bus configuration. This include the bus' clock speed, its voltage, @@ -90,6 +89,10 @@ enum mmc_vdd { vdd_330, vdd_340, vdd_350, vdd_360 }; +enum mmc_vccq { + vccq_120 = 0, vccq_180, vccq_330 +}; + enum mmc_power_mode { power_off = 0, power_up, power_on }; @@ -106,18 +109,28 @@ enum mmc_bus_width { bus_width_1 = 0, bus_width_4 = 2, bus_width_8 = 3 }; +enum mmc_drv_type { + drv_type_b = 0, drv_type_a, drv_type_c, drv_type_d +}; + enum mmc_bus_timing { - bus_timing_normal = 0, bus_timing_hs + bus_timing_normal = 0, bus_timing_hs, bus_timing_uhs_sdr12, + bus_timing_uhs_sdr25, bus_timing_uhs_sdr50, bus_timing_uhs_ddr50, + bus_timing_uhs_sdr104, bus_timing_mmc_ddr52, bus_timing_mmc_hs200, + bus_timing_mmc_hs400, bus_timing_mmc_hs400es, bus_timing_max = + bus_timing_mmc_hs400es }; struct mmc_ios { uint32_t clock; /* Speed of the clock in Hz to move data */ - enum mmc_vdd vdd; /* Voltage to apply to the power pins/ */ + enum mmc_vdd vdd; /* Voltage to apply to the power pins */ + enum mmc_vccq vccq; /* Voltage to use for signaling */ enum mmc_bus_mode bus_mode; enum mmc_chip_select chip_select; enum mmc_bus_width bus_width; enum mmc_power_mode power_mode; enum mmc_bus_timing timing; + enum mmc_drv_type drv_type; }; enum mmc_card_mode { @@ -130,9 +143,33 @@ struct mmc_host { uint32_t host_ocr; uint32_t ocr; uint32_t caps; -#define MMC_CAP_4_BIT_DATA (1 << 0) /* Can do 4-bit data transfers */ -#define MMC_CAP_8_BIT_DATA (1 << 1) /* Can do 8-bit data transfers */ -#define MMC_CAP_HSPEED (1 << 2) /* Can do High Speed transfers */ +#define MMC_CAP_4_BIT_DATA (1 << 0) /* Can do 4-bit data transfers */ +#define MMC_CAP_8_BIT_DATA (1 << 1) /* Can do 8-bit data transfers */ +#define MMC_CAP_HSPEED (1 << 2) /* Can do High Speed transfers */ +#define MMC_CAP_BOOT_NOACC (1 << 4) /* Cannot access boot partitions */ +#define MMC_CAP_WAIT_WHILE_BUSY (1 << 5) /* Host waits for busy responses */ +#define MMC_CAP_UHS_SDR12 (1 << 6) /* Can do UHS SDR12 */ +#define MMC_CAP_UHS_SDR25 (1 << 7) /* Can do UHS SDR25 */ +#define MMC_CAP_UHS_SDR50 (1 << 8) /* Can do UHS SDR50 */ +#define MMC_CAP_UHS_SDR104 (1 << 9) /* Can do UHS SDR104 */ +#define MMC_CAP_UHS_DDR50 (1 << 10) /* Can do UHS DDR50 */ +#define MMC_CAP_MMC_DDR52_120 (1 << 11) /* Can do eMMC DDR52 at 1.2 V */ +#define MMC_CAP_MMC_DDR52_180 (1 << 12) /* Can do eMMC DDR52 at 1.8 V */ +#define MMC_CAP_MMC_DDR52 (MMC_CAP_MMC_DDR52_120 | MMC_CAP_MMC_DDR52_180) +#define MMC_CAP_MMC_HS200_120 (1 << 13) /* Can do eMMC HS200 at 1.2 V */ +#define MMC_CAP_MMC_HS200_180 (1 << 14) /* Can do eMMC HS200 at 1.8 V */ +#define MMC_CAP_MMC_HS200 (MMC_CAP_MMC_HS200_120| MMC_CAP_MMC_HS200_180) +#define MMC_CAP_MMC_HS400_120 (1 << 15) /* Can do eMMC HS400 at 1.2 V */ +#define MMC_CAP_MMC_HS400_180 (1 << 16) /* Can do eMMC HS400 at 1.8 V */ +#define MMC_CAP_MMC_HS400 (MMC_CAP_MMC_HS400_120 | MMC_CAP_MMC_HS400_180) +#define MMC_CAP_MMC_HSX00_120 (MMC_CAP_MMC_HS200_120 | MMC_CAP_MMC_HS400_120) +#define MMC_CAP_MMC_ENH_STROBE (1 << 17) /* Can do eMMC Enhanced Strobe */ +#define MMC_CAP_SIGNALING_120 (1 << 18) /* Can do signaling at 1.2 V */ +#define MMC_CAP_SIGNALING_180 (1 << 19) /* Can do signaling at 1.8 V */ +#define MMC_CAP_SIGNALING_330 (1 << 20) /* Can do signaling at 3.3 V */ +#define MMC_CAP_DRIVER_TYPE_A (1 << 21) /* Can do Driver Type A */ +#define MMC_CAP_DRIVER_TYPE_C (1 << 22) /* Can do Driver Type C */ +#define MMC_CAP_DRIVER_TYPE_D (1 << 23) /* Can do Driver Type D */ enum mmc_card_mode mode; struct mmc_ios ios; /* Current state of the host */ }; @@ -140,4 +177,12 @@ struct mmc_host { extern driver_t mmc_driver; extern devclass_t mmc_devclass; +#define MMC_VERSION 3 + +#define MMC_DECLARE_BRIDGE(name) \ + DRIVER_MODULE(mmc, name, mmc_driver, mmc_devclass, NULL, NULL); \ + MODULE_DEPEND(name, mmc, MMC_VERSION, MMC_VERSION, MMC_VERSION); +#define MMC_DEPEND(name) \ + MODULE_DEPEND(name, mmc, MMC_VERSION, MMC_VERSION, MMC_VERSION); + #endif /* DEV_MMC_BRIDGE_H */ diff --git a/freebsd/sys/dev/mmc/mmc.c b/freebsd/sys/dev/mmc/mmc.c index a3232248..ab494804 100644 --- a/freebsd/sys/dev/mmc/mmc.c +++ b/freebsd/sys/dev/mmc/mmc.c @@ -3,6 +3,7 @@ /*- * Copyright (c) 2006 Bernd Walter. All rights reserved. * Copyright (c) 2006 M. Warner Losh. All rights reserved. + * Copyright (c) 2017 Marius Strobl <marius@FreeBSD.org> * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -67,24 +68,17 @@ __FBSDID("$FreeBSD$"); #include <sys/sysctl.h> #include <sys/time.h> +#include <dev/mmc/bridge.h> +#include <dev/mmc/mmc_private.h> +#include <dev/mmc/mmc_subr.h> #include <dev/mmc/mmcreg.h> #include <dev/mmc/mmcbrvar.h> #include <dev/mmc/mmcvar.h> + #include <rtems/bsd/local/mmcbr_if.h> #include <rtems/bsd/local/mmcbus_if.h> -struct mmc_softc { - device_t dev; - struct mtx sc_mtx; - struct intr_config_hook config_intrhook; - device_t owner; - uint32_t last_rca; - int squelched; /* suppress reporting of (expected) errors */ - int log_count; - struct timeval log_time; -}; - -#define LOG_PPS 5 /* Log no more than 5 errors per second. */ +CTASSERT(bus_timing_max <= sizeof(uint32_t) * NBBY); /* * Per-card data @@ -93,7 +87,7 @@ struct mmc_ivars { uint32_t raw_cid[4]; /* Raw bits of the CID */ uint32_t raw_csd[4]; /* Raw bits of the CSD */ uint32_t raw_scr[2]; /* Raw bits of the SCR */ - uint8_t raw_ext_csd[512]; /* Raw bits of the EXT_CSD */ + uint8_t raw_ext_csd[MMC_EXTCSD_SIZE]; /* Raw bits of the EXT_CSD */ uint32_t raw_sd_status[16]; /* Raw bits of the SD_STATUS */ uint16_t rca; enum mmc_card_mode mode; @@ -103,24 +97,26 @@ struct mmc_ivars { struct mmc_sd_status sd_status; /* SD_STATUS decoded */ u_char read_only; /* True when the device is read-only */ u_char bus_width; /* Bus width to use */ - u_char timing; /* Bus timing support */ u_char high_cap; /* High Capacity card (block addressed) */ uint32_t sec_count; /* Card capacity in 512byte blocks */ + uint32_t timings; /* Mask of bus timings supported */ + uint32_t vccq_120; /* Mask of bus timings at VCCQ of 1.2 V */ + uint32_t vccq_180; /* Mask of bus timings at VCCQ of 1.8 V */ uint32_t tran_speed; /* Max speed in normal mode */ uint32_t hs_tran_speed; /* Max speed in high speed mode */ uint32_t erase_sector; /* Card native erase sector size */ + uint32_t cmd6_time; /* Generic switch timeout [us] */ char card_id_string[64];/* Formatted CID info (serial, MFG, etc) */ char card_sn_string[16];/* Formatted serial # for disk->d_ident */ }; -#define CMD_RETRIES 3 - -#define CARD_ID_FREQUENCY 400000 /* Spec requires 400kHz max during ID phase. */ +#define CMD_RETRIES 3 static SYSCTL_NODE(_hw, OID_AUTO, mmc, CTLFLAG_RD, NULL, "mmc driver"); static int mmc_debug; -SYSCTL_INT(_hw_mmc, OID_AUTO, debug, CTLFLAG_RWTUN, &mmc_debug, 0, "Debug level"); +SYSCTL_INT(_hw_mmc, OID_AUTO, debug, CTLFLAG_RWTUN, &mmc_debug, 0, + "Debug level"); /* bus entry points */ static int mmc_acquire_bus(device_t busdev, device_t dev); @@ -139,14 +135,14 @@ static int mmc_wait_for_request(device_t brdev, device_t reqdev, static int mmc_write_ivar(device_t bus, device_t child, int which, uintptr_t value); -#define MMC_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) +#define MMC_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) #define MMC_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx) -#define MMC_LOCK_INIT(_sc) \ - mtx_init(&_sc->sc_mtx, device_get_nameunit(_sc->dev), \ +#define MMC_LOCK_INIT(_sc) \ + mtx_init(&(_sc)->sc_mtx, device_get_nameunit((_sc)->dev), \ "mmc", MTX_DEF) -#define MMC_LOCK_DESTROY(_sc) mtx_destroy(&_sc->sc_mtx); -#define MMC_ASSERT_LOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_OWNED); -#define MMC_ASSERT_UNLOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_NOTOWNED); +#define MMC_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->sc_mtx); +#define MMC_ASSERT_LOCKED(_sc) mtx_assert(&(_sc)->sc_mtx, MA_OWNED); +#define MMC_ASSERT_UNLOCKED(_sc) mtx_assert(&(_sc)->sc_mtx, MA_NOTOWNED); static int mmc_all_send_cid(struct mmc_softc *sc, uint32_t *rawcid); static void mmc_app_decode_scr(uint32_t *raw_scr, struct mmc_scr *scr); @@ -157,7 +153,8 @@ static int mmc_app_sd_status(struct mmc_softc *sc, uint16_t rca, static int mmc_app_send_scr(struct mmc_softc *sc, uint16_t rca, uint32_t *rawscr); static int mmc_calculate_clock(struct mmc_softc *sc); -static void mmc_decode_cid_mmc(uint32_t *raw_cid, struct mmc_cid *cid); +static void mmc_decode_cid_mmc(uint32_t *raw_cid, struct mmc_cid *cid, + bool is_4_41p); static void mmc_decode_cid_sd(uint32_t *raw_cid, struct mmc_cid *cid); static void mmc_decode_csd_mmc(uint32_t *raw_csd, struct mmc_csd *csd); static void mmc_decode_csd_sd(uint32_t *raw_csd, struct mmc_csd *csd); @@ -183,25 +180,20 @@ static uint32_t mmc_select_vdd(struct mmc_softc *sc, uint32_t ocr); static int mmc_send_app_op_cond(struct mmc_softc *sc, uint32_t ocr, uint32_t *rocr); static int mmc_send_csd(struct mmc_softc *sc, uint16_t rca, uint32_t *rawcsd); -static int mmc_send_ext_csd(struct mmc_softc *sc, uint8_t *rawextcsd); static int mmc_send_if_cond(struct mmc_softc *sc, uint8_t vhs); static int mmc_send_op_cond(struct mmc_softc *sc, uint32_t ocr, uint32_t *rocr); static int mmc_send_relative_addr(struct mmc_softc *sc, uint32_t *resp); -static int mmc_send_status(struct mmc_softc *sc, uint16_t rca, - uint32_t *status); static int mmc_set_blocklen(struct mmc_softc *sc, uint32_t len); -static int mmc_set_card_bus_width(struct mmc_softc *sc, uint16_t rca, - int width); +static int mmc_set_card_bus_width(struct mmc_softc *sc, struct mmc_ivars *ivar); +static int mmc_set_power_class(struct mmc_softc *sc, struct mmc_ivars *ivar); static int mmc_set_relative_addr(struct mmc_softc *sc, uint16_t resp); -static int mmc_set_timing(struct mmc_softc *sc, int timing); -static int mmc_switch(struct mmc_softc *sc, uint8_t set, uint8_t index, - uint8_t value); +static int mmc_set_timing(struct mmc_softc *sc, struct mmc_ivars *ivar, + enum mmc_bus_timing timing); static int mmc_test_bus_width(struct mmc_softc *sc); -static int mmc_wait_for_app_cmd(struct mmc_softc *sc, uint32_t rca, - struct mmc_command *cmd, int retries); -static int mmc_wait_for_cmd(struct mmc_softc *sc, struct mmc_command *cmd, - int retries); +static uint32_t mmc_timing_to_dtr(struct mmc_ivars *ivar, + enum mmc_bus_timing timing); +static const char *mmc_timing_to_string(enum mmc_bus_timing timing); static int mmc_wait_for_command(struct mmc_softc *sc, uint32_t opcode, uint32_t arg, uint32_t flags, uint32_t *resp, int retries); static int mmc_wait_for_req(struct mmc_softc *sc, struct mmc_request *req); @@ -261,7 +253,7 @@ mmc_suspend(device_t dev) err = bus_generic_suspend(dev); if (err) - return (err); + return (err); mmc_power_down(sc); return (0); } @@ -280,8 +272,8 @@ mmc_acquire_bus(device_t busdev, device_t dev) { struct mmc_softc *sc; struct mmc_ivars *ivar; - int err; - int rca; + int err, rca; + enum mmc_bus_timing timing; err = MMCBR_ACQUIRE_HOST(device_get_parent(busdev), busdev); if (err) @@ -300,19 +292,47 @@ mmc_acquire_bus(device_t busdev, device_t dev) * unselect unless the bus code itself wants the mmc * bus, and constantly reselecting causes problems. */ - rca = mmc_get_rca(dev); + ivar = device_get_ivars(dev); + rca = ivar->rca; if (sc->last_rca != rca) { - mmc_select_card(sc, rca); + if (mmc_select_card(sc, rca) != MMC_ERR_NONE) { + device_printf(sc->dev, "Card at relative " + "address %d failed to select.\n", rca); + return (ENXIO); + } sc->last_rca = rca; + timing = mmcbr_get_timing(busdev); /* Prepare bus width for the new card. */ - ivar = device_get_ivars(dev); if (bootverbose || mmc_debug) { device_printf(busdev, - "setting bus width to %d bits\n", + "setting bus width to %d bits %s timing\n", (ivar->bus_width == bus_width_4) ? 4 : - (ivar->bus_width == bus_width_8) ? 8 : 1); + (ivar->bus_width == bus_width_8) ? 8 : 1, + mmc_timing_to_string(timing)); + } + if (mmc_set_card_bus_width(sc, ivar) != MMC_ERR_NONE) { + device_printf(sc->dev, "Card at relative " + "address %d failed to set bus width.\n", + rca); + return (ENXIO); + } + if (isset(&ivar->vccq_120, timing)) + mmcbr_set_vccq(busdev, vccq_120); + else if (isset(&ivar->vccq_180, timing)) + mmcbr_set_vccq(busdev, vccq_180); + else + mmcbr_set_vccq(busdev, vccq_330); + if (mmcbr_switch_vccq(busdev) != 0) { + device_printf(sc->dev, "Failed to set VCCQ " + "for card at relative address %d.\n", rca); + return (ENXIO); + } + if (mmc_set_power_class(sc, ivar) != MMC_ERR_NONE) { + device_printf(sc->dev, "Card at relative " + "address %d failed to set power class.\n", + rca); + return (ENXIO); } - mmc_set_card_bus_width(sc, rca, ivar->bus_width); mmcbr_set_bus_width(busdev, ivar->bus_width); mmcbr_update_ios(busdev); } @@ -409,7 +429,8 @@ mmc_wait_for_req(struct mmc_softc *sc, struct mmc_request *req) } static int -mmc_wait_for_request(device_t brdev, device_t reqdev, struct mmc_request *req) +mmc_wait_for_request(device_t brdev, device_t reqdev __unused, + struct mmc_request *req) { struct mmc_softc *sc = device_get_softc(brdev); @@ -417,74 +438,6 @@ mmc_wait_for_request(device_t brdev, device_t reqdev, struct mmc_request *req) } static int -mmc_wait_for_cmd(struct mmc_softc *sc, struct mmc_command *cmd, int retries) -{ - struct mmc_request mreq; - int err; - - do { - memset(&mreq, 0, sizeof(mreq)); - memset(cmd->resp, 0, sizeof(cmd->resp)); - cmd->retries = 0; /* Retries done here, not in hardware. */ - cmd->mrq = &mreq; - mreq.cmd = cmd; - if (mmc_wait_for_req(sc, &mreq) != 0) - err = MMC_ERR_FAILED; - else - err = cmd->error; - } while (err != MMC_ERR_NONE && retries-- > 0); - - if (err != MMC_ERR_NONE && sc->squelched == 0) { - if (ppsratecheck(&sc->log_time, &sc->log_count, LOG_PPS)) { - device_printf(sc->dev, "CMD%d failed, RESULT: %d\n", - cmd->opcode, err); - } - } - - return (err); -} - -static int -mmc_wait_for_app_cmd(struct mmc_softc *sc, uint32_t rca, - struct mmc_command *cmd, int retries) -{ - struct mmc_command appcmd; - int err; - - /* Squelch error reporting at lower levels, we report below. */ - sc->squelched++; - do { - memset(&appcmd, 0, sizeof(appcmd)); - appcmd.opcode = MMC_APP_CMD; - appcmd.arg = rca << 16; - appcmd.flags = MMC_RSP_R1 | MMC_CMD_AC; - appcmd.data = NULL; - if (mmc_wait_for_cmd(sc, &appcmd, 0) != 0) - err = MMC_ERR_FAILED; - else - err = appcmd.error; - if (err == MMC_ERR_NONE) { - if (!(appcmd.resp[0] & R1_APP_CMD)) - err = MMC_ERR_FAILED; - else if (mmc_wait_for_cmd(sc, cmd, 0) != 0) - err = MMC_ERR_FAILED; - else - err = cmd->error; - } - } while (err != MMC_ERR_NONE && retries-- > 0); - sc->squelched--; - - if (err != MMC_ERR_NONE && sc->squelched == 0) { - if (ppsratecheck(&sc->log_time, &sc->log_count, LOG_PPS)) { - device_printf(sc->dev, "ACMD%d failed, RESULT: %d\n", - cmd->opcode, err); - } - } - - return (err); -} - -static int mmc_wait_for_command(struct mmc_softc *sc, uint32_t opcode, uint32_t arg, uint32_t flags, uint32_t *resp, int retries) { @@ -496,7 +449,7 @@ mmc_wait_for_command(struct mmc_softc *sc, uint32_t opcode, cmd.arg = arg; cmd.flags = flags; cmd.data = NULL; - err = mmc_wait_for_cmd(sc, &cmd, retries); + err = mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, retries); if (err) return (err); if (resp) { @@ -524,7 +477,7 @@ mmc_idle_cards(struct mmc_softc *sc) cmd.arg = 0; cmd.flags = MMC_RSP_NONE | MMC_CMD_BC; cmd.data = NULL; - mmc_wait_for_cmd(sc, &cmd, CMD_RETRIES); + mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, CMD_RETRIES); mmc_ms_delay(1); mmcbr_set_chip_select(dev, cs_dontcare); @@ -545,7 +498,8 @@ mmc_send_app_op_cond(struct mmc_softc *sc, uint32_t ocr, uint32_t *rocr) cmd.data = NULL; for (i = 0; i < 1000; i++) { - err = mmc_wait_for_app_cmd(sc, 0, &cmd, CMD_RETRIES); + err = mmc_wait_for_app_cmd(sc->dev, sc->dev, 0, &cmd, + CMD_RETRIES); if (err != MMC_ERR_NONE) break; if ((cmd.resp[0] & MMC_OCR_CARD_BUSY) || @@ -572,7 +526,7 @@ mmc_send_op_cond(struct mmc_softc *sc, uint32_t ocr, uint32_t *rocr) cmd.data = NULL; for (i = 0; i < 1000; i++) { - err = mmc_wait_for_cmd(sc, &cmd, CMD_RETRIES); + err = mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, CMD_RETRIES); if (err != MMC_ERR_NONE) break; if ((cmd.resp[0] & MMC_OCR_CARD_BUSY) || @@ -598,7 +552,7 @@ mmc_send_if_cond(struct mmc_softc *sc, uint8_t vhs) cmd.flags = MMC_RSP_R7 | MMC_CMD_BCR; cmd.data = NULL; - err = mmc_wait_for_cmd(sc, &cmd, CMD_RETRIES); + err = mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, CMD_RETRIES); return (err); } @@ -606,6 +560,7 @@ static void mmc_power_up(struct mmc_softc *sc) { device_t dev; + enum mmc_vccq vccq; dev = sc->dev; mmcbr_set_vdd(dev, mmc_highest_voltage(mmcbr_get_host_ocr(dev))); @@ -615,9 +570,14 @@ mmc_power_up(struct mmc_softc *sc) mmcbr_set_power_mode(dev, power_up); mmcbr_set_clock(dev, 0); mmcbr_update_ios(dev); + for (vccq = vccq_330; ; vccq--) { + mmcbr_set_vccq(dev, vccq); + if (mmcbr_switch_vccq(dev) == 0 || vccq == vccq_120) + break; + } mmc_ms_delay(1); - mmcbr_set_clock(dev, CARD_ID_FREQUENCY); + mmcbr_set_clock(dev, SD_MMC_CARD_ID_FREQUENCY); mmcbr_set_timing(dev, bus_timing_normal); mmcbr_set_power_mode(dev, power_on); mmcbr_update_ios(dev); @@ -649,24 +609,6 @@ mmc_select_card(struct mmc_softc *sc, uint16_t rca) } static int -mmc_switch(struct mmc_softc *sc, uint8_t set, uint8_t index, uint8_t value) -{ - struct mmc_command cmd; - int err; - - memset(&cmd, 0, sizeof(cmd)); - cmd.opcode = MMC_SWITCH_FUNC; - cmd.arg = (MMC_SWITCH_FUNC_WR << 24) | - (index << 16) | - (value << 8) | - set; - cmd.flags = MMC_RSP_R1B | MMC_CMD_AC; - cmd.data = NULL; - err = mmc_wait_for_cmd(sc, &cmd, CMD_RETRIES); - return (err); -} - -static int mmc_sd_switch(struct mmc_softc *sc, uint8_t mode, uint8_t grp, uint8_t value, uint8_t *res) { @@ -690,12 +632,12 @@ mmc_sd_switch(struct mmc_softc *sc, uint8_t mode, uint8_t grp, uint8_t value, data.len = 64; data.flags = MMC_DATA_READ; - err = mmc_wait_for_cmd(sc, &cmd, CMD_RETRIES); + err = mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, CMD_RETRIES); return (err); } static int -mmc_set_card_bus_width(struct mmc_softc *sc, uint16_t rca, int width) +mmc_set_card_bus_width(struct mmc_softc *sc, struct mmc_ivars *ivar) { struct mmc_command cmd; int err; @@ -706,13 +648,14 @@ mmc_set_card_bus_width(struct mmc_softc *sc, uint16_t rca, int width) cmd.opcode = ACMD_SET_CLR_CARD_DETECT; cmd.flags = MMC_RSP_R1 | MMC_CMD_AC; cmd.arg = SD_CLR_CARD_DETECT; - err = mmc_wait_for_app_cmd(sc, rca, &cmd, CMD_RETRIES); + err = mmc_wait_for_app_cmd(sc->dev, sc->dev, ivar->rca, &cmd, + CMD_RETRIES); if (err != 0) return (err); memset(&cmd, 0, sizeof(cmd)); cmd.opcode = ACMD_SET_BUS_WIDTH; cmd.flags = MMC_RSP_R1 | MMC_CMD_AC; - switch (width) { + switch (ivar->bus_width) { case bus_width_1: cmd.arg = SD_BUS_WIDTH_1; break; @@ -722,64 +665,196 @@ mmc_set_card_bus_width(struct mmc_softc *sc, uint16_t rca, int width) default: return (MMC_ERR_INVALID); } - err = mmc_wait_for_app_cmd(sc, rca, &cmd, CMD_RETRIES); + err = mmc_wait_for_app_cmd(sc->dev, sc->dev, ivar->rca, &cmd, + CMD_RETRIES); } else { - switch (width) { + switch (ivar->bus_width) { case bus_width_1: value = EXT_CSD_BUS_WIDTH_1; break; case bus_width_4: - value = EXT_CSD_BUS_WIDTH_4; + switch (mmcbr_get_timing(sc->dev)) { + case bus_timing_mmc_ddr52: + case bus_timing_mmc_hs200: + case bus_timing_mmc_hs400: + case bus_timing_mmc_hs400es: + value = EXT_CSD_BUS_WIDTH_4_DDR; + break; + default: + value = EXT_CSD_BUS_WIDTH_4; + break; + } break; case bus_width_8: - value = EXT_CSD_BUS_WIDTH_8; + switch (mmcbr_get_timing(sc->dev)) { + case bus_timing_mmc_ddr52: + case bus_timing_mmc_hs200: + case bus_timing_mmc_hs400: + case bus_timing_mmc_hs400es: + value = EXT_CSD_BUS_WIDTH_8_DDR; + break; + default: + value = EXT_CSD_BUS_WIDTH_8; + break; + } break; default: return (MMC_ERR_INVALID); } - err = mmc_switch(sc, EXT_CSD_CMD_SET_NORMAL, EXT_CSD_BUS_WIDTH, - value); + err = mmc_switch(sc->dev, sc->dev, ivar->rca, + EXT_CSD_CMD_SET_NORMAL, EXT_CSD_BUS_WIDTH, value, + ivar->cmd6_time, true); } return (err); } static int -mmc_set_timing(struct mmc_softc *sc, int timing) +mmc_set_power_class(struct mmc_softc *sc, struct mmc_ivars *ivar) { - int err; - uint8_t value; - u_char switch_res[64]; + device_t dev; + const uint8_t *ext_csd; + uint32_t clock; + uint8_t value; - switch (timing) { - case bus_timing_normal: - value = 0; + dev = sc->dev; + if (mmcbr_get_mode(dev) != mode_mmc || ivar->csd.spec_vers < 4) + return (MMC_ERR_NONE); + + value = 0; + ext_csd = ivar->raw_ext_csd; + clock = mmcbr_get_clock(dev); + switch (1 << mmcbr_get_vdd(dev)) { + case MMC_OCR_LOW_VOLTAGE: + if (clock <= MMC_TYPE_HS_26_MAX) + value = ext_csd[EXT_CSD_PWR_CL_26_195]; + else if (clock <= MMC_TYPE_HS_52_MAX) { + if (mmcbr_get_timing(dev) >= bus_timing_mmc_ddr52 && + ivar->bus_width >= bus_width_4) + value = ext_csd[EXT_CSD_PWR_CL_52_195_DDR]; + else + value = ext_csd[EXT_CSD_PWR_CL_52_195]; + } else if (clock <= MMC_TYPE_HS200_HS400ES_MAX) + value = ext_csd[EXT_CSD_PWR_CL_200_195]; break; - case bus_timing_hs: - value = 1; + case MMC_OCR_270_280: + case MMC_OCR_280_290: + case MMC_OCR_290_300: + case MMC_OCR_300_310: + case MMC_OCR_310_320: + case MMC_OCR_320_330: + case MMC_OCR_330_340: + case MMC_OCR_340_350: + case MMC_OCR_350_360: + if (clock <= MMC_TYPE_HS_26_MAX) + value = ext_csd[EXT_CSD_PWR_CL_26_360]; + else if (clock <= MMC_TYPE_HS_52_MAX) { + if (mmcbr_get_timing(dev) == bus_timing_mmc_ddr52 && + ivar->bus_width >= bus_width_4) + value = ext_csd[EXT_CSD_PWR_CL_52_360_DDR]; + else + value = ext_csd[EXT_CSD_PWR_CL_52_360]; + } else if (clock <= MMC_TYPE_HS200_HS400ES_MAX) { + if (ivar->bus_width == bus_width_8) + value = ext_csd[EXT_CSD_PWR_CL_200_360_DDR]; + else + value = ext_csd[EXT_CSD_PWR_CL_200_360]; + } break; default: + device_printf(dev, "No power class support for VDD 0x%x\n", + 1 << mmcbr_get_vdd(dev)); return (MMC_ERR_INVALID); } - if (mmcbr_get_mode(sc->dev) == mode_sd) + + if (ivar->bus_width == bus_width_8) + value = (value & EXT_CSD_POWER_CLASS_8BIT_MASK) >> + EXT_CSD_POWER_CLASS_8BIT_SHIFT; + else + value = (value & EXT_CSD_POWER_CLASS_4BIT_MASK) >> + EXT_CSD_POWER_CLASS_4BIT_SHIFT; + + if (value == 0) + return (MMC_ERR_NONE); + + return (mmc_switch(dev, dev, ivar->rca, EXT_CSD_CMD_SET_NORMAL, + EXT_CSD_POWER_CLASS, value, ivar->cmd6_time, true)); +} + +static int +mmc_set_timing(struct mmc_softc *sc, struct mmc_ivars *ivar, + enum mmc_bus_timing timing) +{ + u_char switch_res[64]; + uint8_t value; + int err; + + if (mmcbr_get_mode(sc->dev) == mode_sd) { + switch (timing) { + case bus_timing_normal: + value = SD_SWITCH_NORMAL_MODE; + break; + case bus_timing_hs: + value = SD_SWITCH_HS_MODE; + break; + default: + return (MMC_ERR_INVALID); + } err = mmc_sd_switch(sc, SD_SWITCH_MODE_SET, SD_SWITCH_GROUP1, value, switch_res); - else - err = mmc_switch(sc, EXT_CSD_CMD_SET_NORMAL, - EXT_CSD_HS_TIMING, value); + if (err != MMC_ERR_NONE) + return (err); + if ((switch_res[16] & 0xf) != value) + return (MMC_ERR_FAILED); + mmcbr_set_timing(sc->dev, timing); + mmcbr_update_ios(sc->dev); + } else { + switch (timing) { + case bus_timing_normal: + value = EXT_CSD_HS_TIMING_BC; + break; + case bus_timing_hs: + case bus_timing_mmc_ddr52: + value = EXT_CSD_HS_TIMING_HS; + break; + default: + return (MMC_ERR_INVALID); + } + err = mmc_switch(sc->dev, sc->dev, ivar->rca, + EXT_CSD_CMD_SET_NORMAL, EXT_CSD_HS_TIMING, value, + ivar->cmd6_time, false); + if (err != MMC_ERR_NONE) + return (err); + mmcbr_set_timing(sc->dev, timing); + mmcbr_update_ios(sc->dev); + err = mmc_switch_status(sc->dev, sc->dev, ivar->rca, + ivar->cmd6_time); + } return (err); } +static const uint8_t p8[8] = { + 0x55, 0xAA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +static const uint8_t p8ok[8] = { + 0xAA, 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +static const uint8_t p4[4] = { + 0x5A, 0x00, 0x00, 0x00 +}; + +static const uint8_t p4ok[4] = { + 0xA5, 0x00, 0x00, 0x00 +}; + static int mmc_test_bus_width(struct mmc_softc *sc) { struct mmc_command cmd; struct mmc_data data; - int err; uint8_t buf[8]; - uint8_t p8[8] = { 0x55, 0xAA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; - uint8_t p8ok[8] = { 0xAA, 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; - uint8_t p4[4] = { 0x5A, 0x00, 0x00, 0x00, }; - uint8_t p4ok[4] = { 0xA5, 0x00, 0x00, 0x00, }; + int err; if (mmcbr_get_caps(sc->dev) & MMC_CAP_8_BIT_DATA) { mmcbr_set_bus_width(sc->dev, bus_width_8); @@ -793,10 +868,10 @@ mmc_test_bus_width(struct mmc_softc *sc) cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC; cmd.data = &data; - data.data = p8; + data.data = __DECONST(void *, p8); data.len = 8; data.flags = MMC_DATA_WRITE; - mmc_wait_for_cmd(sc, &cmd, 0); + mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, 0); memset(&cmd, 0, sizeof(cmd)); memset(&data, 0, sizeof(data)); @@ -808,7 +883,7 @@ mmc_test_bus_width(struct mmc_softc *sc) data.data = buf; data.len = 8; data.flags = MMC_DATA_READ; - err = mmc_wait_for_cmd(sc, &cmd, 0); + err = mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, 0); sc->squelched--; mmcbr_set_bus_width(sc->dev, bus_width_1); @@ -830,10 +905,10 @@ mmc_test_bus_width(struct mmc_softc *sc) cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC; cmd.data = &data; - data.data = p4; + data.data = __DECONST(void *, p4); data.len = 4; data.flags = MMC_DATA_WRITE; - mmc_wait_for_cmd(sc, &cmd, 0); + mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, 0); memset(&cmd, 0, sizeof(cmd)); memset(&data, 0, sizeof(data)); @@ -845,7 +920,7 @@ mmc_test_bus_width(struct mmc_softc *sc) data.data = buf; data.len = 4; data.flags = MMC_DATA_READ; - err = mmc_wait_for_cmd(sc, &cmd, 0); + err = mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, 0); sc->squelched--; mmcbr_set_bus_width(sc->dev, bus_width_1); @@ -863,6 +938,7 @@ mmc_get_bits(uint32_t *bits, int bit_len, int start, int size) const int i = (bit_len / 32) - (start / 32) - 1; const int shift = start & 31; uint32_t retval = bits[i] >> shift; + if (size + shift > 32) retval |= bits[i - 1] << (32 - shift); return (retval & ((1llu << size) - 1)); @@ -887,7 +963,7 @@ mmc_decode_cid_sd(uint32_t *raw_cid, struct mmc_cid *cid) } static void -mmc_decode_cid_mmc(uint32_t *raw_cid, struct mmc_cid *cid) +mmc_decode_cid_mmc(uint32_t *raw_cid, struct mmc_cid *cid, bool is_4_41p) { int i; @@ -901,7 +977,11 @@ mmc_decode_cid_mmc(uint32_t *raw_cid, struct mmc_cid *cid) cid->prv = mmc_get_bits(raw_cid, 128, 48, 8); cid->psn = mmc_get_bits(raw_cid, 128, 16, 32); cid->mdt_month = mmc_get_bits(raw_cid, 128, 12, 4); - cid->mdt_year = mmc_get_bits(raw_cid, 128, 8, 4) + 1997; + cid->mdt_year = mmc_get_bits(raw_cid, 128, 8, 4); + if (is_4_41p) + cid->mdt_year += 2013; + else + cid->mdt_year += 1997; } static void @@ -982,10 +1062,14 @@ mmc_decode_csd_sd(uint32_t *raw_csd, struct mmc_csd *csd) csd->write_blk_misalign = mmc_get_bits(raw_csd, 128, 78, 1); csd->read_blk_misalign = mmc_get_bits(raw_csd, 128, 77, 1); csd->dsr_imp = mmc_get_bits(raw_csd, 128, 76, 1); - csd->vdd_r_curr_min = cur_min[mmc_get_bits(raw_csd, 128, 59, 3)]; - csd->vdd_r_curr_max = cur_max[mmc_get_bits(raw_csd, 128, 56, 3)]; - csd->vdd_w_curr_min = cur_min[mmc_get_bits(raw_csd, 128, 53, 3)]; - csd->vdd_w_curr_max = cur_max[mmc_get_bits(raw_csd, 128, 50, 3)]; + csd->vdd_r_curr_min = + cur_min[mmc_get_bits(raw_csd, 128, 59, 3)]; + csd->vdd_r_curr_max = + cur_max[mmc_get_bits(raw_csd, 128, 56, 3)]; + csd->vdd_w_curr_min = + cur_min[mmc_get_bits(raw_csd, 128, 53, 3)]; + csd->vdd_w_curr_max = + cur_max[mmc_get_bits(raw_csd, 128, 50, 3)]; m = mmc_get_bits(raw_csd, 128, 62, 12); e = mmc_get_bits(raw_csd, 128, 47, 3); csd->capacity = ((1 + m) << (e + 2)) * csd->read_bl_len; @@ -1010,8 +1094,8 @@ mmc_decode_csd_sd(uint32_t *raw_csd, struct mmc_csd *csd) csd->write_blk_misalign = mmc_get_bits(raw_csd, 128, 78, 1); csd->read_blk_misalign = mmc_get_bits(raw_csd, 128, 77, 1); csd->dsr_imp = mmc_get_bits(raw_csd, 128, 76, 1); - csd->capacity = ((uint64_t)mmc_get_bits(raw_csd, 128, 48, 22) + 1) * - 512 * 1024; + csd->capacity = ((uint64_t)mmc_get_bits(raw_csd, 128, 48, 22) + + 1) * 512 * 1024; csd->erase_blk_en = mmc_get_bits(raw_csd, 128, 46, 1); csd->erase_sector = mmc_get_bits(raw_csd, 128, 39, 7) + 1; csd->wp_grp_size = mmc_get_bits(raw_csd, 128, 32, 7); @@ -1109,7 +1193,7 @@ mmc_all_send_cid(struct mmc_softc *sc, uint32_t *rawcid) cmd.arg = 0; cmd.flags = MMC_RSP_R2 | MMC_CMD_BCR; cmd.data = NULL; - err = mmc_wait_for_cmd(sc, &cmd, CMD_RETRIES); + err = mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, CMD_RETRIES); memcpy(rawcid, cmd.resp, 4 * sizeof(uint32_t)); return (err); } @@ -1125,7 +1209,7 @@ mmc_send_csd(struct mmc_softc *sc, uint16_t rca, uint32_t *rawcsd) cmd.arg = rca << 16; cmd.flags = MMC_RSP_R2 | MMC_CMD_BCR; cmd.data = NULL; - err = mmc_wait_for_cmd(sc, &cmd, CMD_RETRIES); + err = mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, CMD_RETRIES); memcpy(rawcsd, cmd.resp, 4 * sizeof(uint32_t)); return (err); } @@ -1150,42 +1234,18 @@ mmc_app_send_scr(struct mmc_softc *sc, uint16_t rca, uint32_t *rawscr) data.len = 8; data.flags = MMC_DATA_READ; - err = mmc_wait_for_app_cmd(sc, rca, &cmd, CMD_RETRIES); + err = mmc_wait_for_app_cmd(sc->dev, sc->dev, rca, &cmd, CMD_RETRIES); rawscr[0] = be32toh(rawscr[0]); rawscr[1] = be32toh(rawscr[1]); return (err); } static int -mmc_send_ext_csd(struct mmc_softc *sc, uint8_t *rawextcsd) -{ - int err; - struct mmc_command cmd; - struct mmc_data data; - - memset(&cmd, 0, sizeof(cmd)); - memset(&data, 0, sizeof(data)); - - memset(rawextcsd, 0, 512); - cmd.opcode = MMC_SEND_EXT_CSD; - cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC; - cmd.arg = 0; - cmd.data = &data; - - data.data = rawextcsd; - data.len = 512; - data.flags = MMC_DATA_READ; - - err = mmc_wait_for_cmd(sc, &cmd, CMD_RETRIES); - return (err); -} - -static int mmc_app_sd_status(struct mmc_softc *sc, uint16_t rca, uint32_t *rawsdstatus) { - int err, i; struct mmc_command cmd; struct mmc_data data; + int err, i; memset(&cmd, 0, sizeof(cmd)); memset(&data, 0, sizeof(data)); @@ -1200,7 +1260,7 @@ mmc_app_sd_status(struct mmc_softc *sc, uint16_t rca, uint32_t *rawsdstatus) data.len = 64; data.flags = MMC_DATA_READ; - err = mmc_wait_for_app_cmd(sc, rca, &cmd, CMD_RETRIES); + err = mmc_wait_for_app_cmd(sc->dev, sc->dev, rca, &cmd, CMD_RETRIES); for (i = 0; i < 16; i++) rawsdstatus[i] = be32toh(rawsdstatus[i]); return (err); @@ -1217,7 +1277,7 @@ mmc_set_relative_addr(struct mmc_softc *sc, uint16_t resp) cmd.arg = resp << 16; cmd.flags = MMC_RSP_R6 | MMC_CMD_BCR; cmd.data = NULL; - err = mmc_wait_for_cmd(sc, &cmd, CMD_RETRIES); + err = mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, CMD_RETRIES); return (err); } @@ -1232,54 +1292,102 @@ mmc_send_relative_addr(struct mmc_softc *sc, uint32_t *resp) cmd.arg = 0; cmd.flags = MMC_RSP_R6 | MMC_CMD_BCR; cmd.data = NULL; - err = mmc_wait_for_cmd(sc, &cmd, CMD_RETRIES); + err = mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, CMD_RETRIES); *resp = cmd.resp[0]; return (err); } static int -mmc_send_status(struct mmc_softc *sc, uint16_t rca, uint32_t *status) +mmc_set_blocklen(struct mmc_softc *sc, uint32_t len) { struct mmc_command cmd; int err; memset(&cmd, 0, sizeof(cmd)); - cmd.opcode = MMC_SEND_STATUS; - cmd.arg = rca << 16; + cmd.opcode = MMC_SET_BLOCKLEN; + cmd.arg = len; cmd.flags = MMC_RSP_R1 | MMC_CMD_AC; cmd.data = NULL; - err = mmc_wait_for_cmd(sc, &cmd, CMD_RETRIES); - *status = cmd.resp[0]; + err = mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, CMD_RETRIES); return (err); } -static int -mmc_set_blocklen(struct mmc_softc *sc, uint32_t len) +static uint32_t +mmc_timing_to_dtr(struct mmc_ivars *ivar, enum mmc_bus_timing timing) { - struct mmc_command cmd; - int err; - memset(&cmd, 0, sizeof(cmd)); - cmd.opcode = MMC_SET_BLOCKLEN; - cmd.arg = len; - cmd.flags = MMC_RSP_R1 | MMC_CMD_AC; - cmd.data = NULL; - err = mmc_wait_for_cmd(sc, &cmd, CMD_RETRIES); - return (err); + switch (timing) { + case bus_timing_normal: + return (ivar->tran_speed); + case bus_timing_hs: + return (ivar->hs_tran_speed); + case bus_timing_uhs_sdr12: + return (SD_SDR12_MAX); + case bus_timing_uhs_sdr25: + return (SD_SDR25_MAX); + case bus_timing_uhs_ddr50: + return (SD_DDR50_MAX); + case bus_timing_uhs_sdr50: + return (SD_SDR50_MAX); + case bus_timing_uhs_sdr104: + return (SD_SDR104_MAX); + case bus_timing_mmc_ddr52: + return (MMC_TYPE_DDR52_MAX); + case bus_timing_mmc_hs200: + case bus_timing_mmc_hs400: + case bus_timing_mmc_hs400es: + return (MMC_TYPE_HS200_HS400ES_MAX); + } + return (0); +} + +static const char * +mmc_timing_to_string(enum mmc_bus_timing timing) +{ + + switch (timing) { + case bus_timing_normal: + return ("normal speed"); + case bus_timing_hs: + return ("high speed"); + case bus_timing_uhs_sdr12: + case bus_timing_uhs_sdr25: + case bus_timing_uhs_sdr50: + case bus_timing_uhs_sdr104: + return ("single data rate"); + case bus_timing_uhs_ddr50: + case bus_timing_mmc_ddr52: + return ("dual data rate"); + case bus_timing_mmc_hs200: + return ("HS200"); + case bus_timing_mmc_hs400: + return ("HS400"); + case bus_timing_mmc_hs400es: + return ("HS400 with enhanced strobe"); + } + return (""); } static void mmc_log_card(device_t dev, struct mmc_ivars *ivar, int newcard) { + enum mmc_bus_timing max_timing, timing; + device_printf(dev, "Card at relative address 0x%04x%s:\n", ivar->rca, newcard ? " added" : ""); device_printf(dev, " card: %s\n", ivar->card_id_string); - device_printf(dev, " bus: %ubit, %uMHz%s\n", + max_timing = bus_timing_normal; + for (timing = bus_timing_max; timing > bus_timing_normal; timing--) { + if (isset(&ivar->timings, timing)) { + max_timing = timing; + break; + } + } + device_printf(dev, " bus: %ubit, %uMHz (%s timing)\n", (ivar->bus_width == bus_width_1 ? 1 : (ivar->bus_width == bus_width_4 ? 4 : 8)), - (ivar->timing == bus_timing_hs ? - ivar->hs_tran_speed : ivar->tran_speed) / 1000000, - ivar->timing == bus_timing_hs ? ", high speed timing" : ""); + mmc_timing_to_dtr(ivar, timing) / 1000000, + mmc_timing_to_string(timing)); device_printf(dev, " memory: %u blocks, erase sector %u blocks%s\n", ivar->sec_count, ivar->erase_sector, ivar->read_only ? ", read-only" : ""); @@ -1288,14 +1396,16 @@ mmc_log_card(device_t dev, struct mmc_ivars *ivar, int newcard) static void mmc_discover_cards(struct mmc_softc *sc) { + u_char switch_res[64]; + uint32_t raw_cid[4]; struct mmc_ivars *ivar = NULL; device_t *devlist; - int err, i, devcount, newcard; - uint32_t raw_cid[4], resp, sec_count, status; device_t child; + int devcount, err, host_caps, i, newcard; + uint32_t resp, sec_count, status; uint16_t rca = 2; - u_char switch_res[64]; + host_caps = mmcbr_get_caps(sc->dev); if (bootverbose || mmc_debug) device_printf(sc->dev, "Probing cards\n"); while (1) { @@ -1309,18 +1419,21 @@ mmc_discover_cards(struct mmc_softc *sc) break; } newcard = 1; - if ((err = device_get_children(sc->dev, &devlist, &devcount)) != 0) + if ((err = device_get_children(sc->dev, &devlist, + &devcount)) != 0) return; for (i = 0; i < devcount; i++) { ivar = device_get_ivars(devlist[i]); - if (memcmp(ivar->raw_cid, raw_cid, sizeof(raw_cid)) == 0) { + if (memcmp(ivar->raw_cid, raw_cid, sizeof(raw_cid)) == + 0) { newcard = 0; break; } } free(devlist, M_TEMP); if (bootverbose || mmc_debug) { - device_printf(sc->dev, "%sard detected (CID %08x%08x%08x%08x)\n", + device_printf(sc->dev, + "%sard detected (CID %08x%08x%08x%08x)\n", newcard ? "New c" : "C", raw_cid[0], raw_cid[1], raw_cid[2], raw_cid[3]); } @@ -1332,14 +1445,24 @@ mmc_discover_cards(struct mmc_softc *sc) if (mmcbr_get_ro(sc->dev)) ivar->read_only = 1; ivar->bus_width = bus_width_1; - ivar->timing = bus_timing_normal; + setbit(&ivar->timings, bus_timing_normal); ivar->mode = mmcbr_get_mode(sc->dev); if (ivar->mode == mode_sd) { mmc_decode_cid_sd(ivar->raw_cid, &ivar->cid); - mmc_send_relative_addr(sc, &resp); + err = mmc_send_relative_addr(sc, &resp); + if (err != MMC_ERR_NONE) { + device_printf(sc->dev, + "Error getting RCA %d\n", err); + break; + } ivar->rca = resp >> 16; /* Get card CSD. */ - mmc_send_csd(sc, ivar->rca, ivar->raw_csd); + err = mmc_send_csd(sc, ivar->rca, ivar->raw_csd); + if (err != MMC_ERR_NONE) { + device_printf(sc->dev, + "Error getting CSD %d\n", err); + break; + } if (bootverbose || mmc_debug) device_printf(sc->dev, "%sard detected (CSD %08x%08x%08x%08x)\n", @@ -1354,7 +1477,8 @@ mmc_discover_cards(struct mmc_softc *sc) ivar->erase_sector = ivar->csd.erase_sector * ivar->csd.write_bl_len / MMC_SECTOR_SIZE; - err = mmc_send_status(sc, ivar->rca, &status); + err = mmc_send_status(sc->dev, sc->dev, ivar->rca, + &status); if (err != MMC_ERR_NONE) { device_printf(sc->dev, "Error reading card status %d\n", err); @@ -1366,19 +1490,30 @@ mmc_discover_cards(struct mmc_softc *sc) break; } - /* Get card SCR. Card must be selected to fetch it. */ - mmc_select_card(sc, ivar->rca); - mmc_app_send_scr(sc, ivar->rca, ivar->raw_scr); + /* Get card SCR. Card must be selected to fetch it. */ + err = mmc_select_card(sc, ivar->rca); + if (err != MMC_ERR_NONE) { + device_printf(sc->dev, + "Error selecting card %d\n", err); + break; + } + err = mmc_app_send_scr(sc, ivar->rca, ivar->raw_scr); + if (err != MMC_ERR_NONE) { + device_printf(sc->dev, + "Error reading SCR %d\n", err); + break; + } mmc_app_decode_scr(ivar->raw_scr, &ivar->scr); /* Get card switch capabilities (command class 10). */ if ((ivar->scr.sda_vsn >= 1) && - (ivar->csd.ccc & (1<<10))) { - mmc_sd_switch(sc, SD_SWITCH_MODE_CHECK, + (ivar->csd.ccc & (1 << 10))) { + err = mmc_sd_switch(sc, SD_SWITCH_MODE_CHECK, SD_SWITCH_GROUP1, SD_SWITCH_NOCHANGE, switch_res); - if (switch_res[13] & 2) { - ivar->timing = bus_timing_hs; - ivar->hs_tran_speed = SD_MAX_HS; + if (err == MMC_ERR_NONE && + switch_res[13] & (1 << SD_SWITCH_HS_MODE)) { + setbit(&ivar->timings, bus_timing_hs); + ivar->hs_tran_speed = SD_HS_MAX; } } @@ -1388,15 +1523,16 @@ mmc_discover_cards(struct mmc_softc *sc) * commands, although the state tables / diagrams in the * standard suggest they go back to the transfer state. * Other cards don't become deselected, and if we - * atttempt to blindly re-select them, we get timeout + * attempt to blindly re-select them, we get timeout * errors from some controllers. So we deselect then * reselect to handle all situations. The only thing we * use from the sd_status is the erase sector size, but * it is still nice to get that right. */ mmc_select_card(sc, 0); - mmc_select_card(sc, ivar->rca); - mmc_app_sd_status(sc, ivar->rca, ivar->raw_sd_status); + (void)mmc_select_card(sc, ivar->rca); + (void)mmc_app_sd_status(sc, ivar->rca, + ivar->raw_sd_status); mmc_app_decode_sd_status(ivar->raw_sd_status, &ivar->sd_status); if (ivar->sd_status.au_size != 0) { @@ -1404,7 +1540,7 @@ mmc_discover_cards(struct mmc_softc *sc) 16 << ivar->sd_status.au_size; } /* Find max supported bus width. */ - if ((mmcbr_get_caps(sc->dev) & MMC_CAP_4_BIT_DATA) && + if ((host_caps & MMC_CAP_4_BIT_DATA) && (ivar->scr.bus_widths & SD_SCR_BUS_WIDTH_4)) ivar->bus_width = bus_width_4; @@ -1433,11 +1569,18 @@ mmc_discover_cards(struct mmc_softc *sc) mmc_select_card(sc, 0); return; } - mmc_decode_cid_mmc(ivar->raw_cid, &ivar->cid); ivar->rca = rca++; - mmc_set_relative_addr(sc, ivar->rca); + err = mmc_set_relative_addr(sc, ivar->rca); + if (err != MMC_ERR_NONE) { + device_printf(sc->dev, "Error setting RCA %d\n", err); + break; + } /* Get card CSD. */ - mmc_send_csd(sc, ivar->rca, ivar->raw_csd); + err = mmc_send_csd(sc, ivar->rca, ivar->raw_csd); + if (err != MMC_ERR_NONE) { + device_printf(sc->dev, "Error getting CSD %d\n", err); + break; + } if (bootverbose || mmc_debug) device_printf(sc->dev, "%sard detected (CSD %08x%08x%08x%08x)\n", @@ -1451,7 +1594,7 @@ mmc_discover_cards(struct mmc_softc *sc) ivar->erase_sector = ivar->csd.erase_sector * ivar->csd.write_bl_len / MMC_SECTOR_SIZE; - err = mmc_send_status(sc, ivar->rca, &status); + err = mmc_send_status(sc->dev, sc->dev, ivar->rca, &status); if (err != MMC_ERR_NONE) { device_printf(sc->dev, "Error reading card status %d\n", err); @@ -1463,11 +1606,22 @@ mmc_discover_cards(struct mmc_softc *sc) break; } - mmc_select_card(sc, ivar->rca); + err = mmc_select_card(sc, ivar->rca); + if (err != MMC_ERR_NONE) { + device_printf(sc->dev, "Error selecting card %d\n", + err); + break; + } - /* Only MMC >= 4.x cards support EXT_CSD. */ + /* Only MMC >= 4.x devices support EXT_CSD. */ if (ivar->csd.spec_vers >= 4) { - mmc_send_ext_csd(sc, ivar->raw_ext_csd); + err = mmc_send_ext_csd(sc->dev, sc->dev, + ivar->raw_ext_csd); + if (err != MMC_ERR_NONE) { + device_printf(sc->dev, + "Error reading EXT_CSD %d\n", err); + break; + } /* Handle extended capacity from EXT_CSD */ sec_count = ivar->raw_ext_csd[EXT_CSD_SEC_CNT] + (ivar->raw_ext_csd[EXT_CSD_SEC_CNT + 1] << 8) + @@ -1477,28 +1631,54 @@ mmc_discover_cards(struct mmc_softc *sc) ivar->sec_count = sec_count; ivar->high_cap = 1; } - /* Get card speed in high speed mode. */ - ivar->timing = bus_timing_hs; - if (ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] - & EXT_CSD_CARD_TYPE_52) - ivar->hs_tran_speed = MMC_TYPE_52_MAX_HS; - else if (ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] - & EXT_CSD_CARD_TYPE_26) - ivar->hs_tran_speed = MMC_TYPE_26_MAX_HS; - else - ivar->hs_tran_speed = ivar->tran_speed; + /* Get device speeds beyond normal mode. */ + if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] & + EXT_CSD_CARD_TYPE_HS_52) != 0) { + setbit(&ivar->timings, bus_timing_hs); + ivar->hs_tran_speed = MMC_TYPE_HS_52_MAX; + } else if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] & + EXT_CSD_CARD_TYPE_HS_26) != 0) { + setbit(&ivar->timings, bus_timing_hs); + ivar->hs_tran_speed = MMC_TYPE_HS_26_MAX; + } + if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] & + EXT_CSD_CARD_TYPE_DDR_52_1_2V) != 0 && + (host_caps & MMC_CAP_SIGNALING_120) != 0) { + setbit(&ivar->timings, bus_timing_mmc_ddr52); + setbit(&ivar->vccq_120, bus_timing_mmc_ddr52); + } + if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] & + EXT_CSD_CARD_TYPE_DDR_52_1_8V) != 0 && + (host_caps & MMC_CAP_SIGNALING_180) != 0) { + setbit(&ivar->timings, bus_timing_mmc_ddr52); + setbit(&ivar->vccq_180, bus_timing_mmc_ddr52); + } + /* + * Determine generic switch timeout (provided in + * units of 10 ms), defaulting to 500 ms. + */ + ivar->cmd6_time = 500 * 1000; + if (ivar->csd.spec_vers >= 6) + ivar->cmd6_time = 10 * + ivar->raw_ext_csd[EXT_CSD_GEN_CMD6_TIME]; /* Find max supported bus width. */ ivar->bus_width = mmc_test_bus_width(sc); /* Handle HC erase sector size. */ if (ivar->raw_ext_csd[EXT_CSD_ERASE_GRP_SIZE] != 0) { ivar->erase_sector = 1024 * ivar->raw_ext_csd[EXT_CSD_ERASE_GRP_SIZE]; - mmc_switch(sc, EXT_CSD_CMD_SET_NORMAL, - EXT_CSD_ERASE_GRP_DEF, 1); + err = mmc_switch(sc->dev, sc->dev, ivar->rca, + EXT_CSD_CMD_SET_NORMAL, + EXT_CSD_ERASE_GRP_DEF, + EXT_CSD_ERASE_GRP_DEF_EN, + ivar->cmd6_time, true); + if (err != MMC_ERR_NONE) { + device_printf(sc->dev, + "Error setting erase group %d\n", + err); + break; + } } - } else { - ivar->bus_width = bus_width_1; - ivar->timing = bus_timing_normal; } /* @@ -1513,6 +1693,8 @@ mmc_discover_cards(struct mmc_softc *sc) ivar->csd.write_bl_len != MMC_SECTOR_SIZE) mmc_set_blocklen(sc, MMC_SECTOR_SIZE); + mmc_decode_cid_mmc(ivar->raw_cid, &ivar->cid, + ivar->raw_ext_csd[EXT_CSD_REV] >= 5); mmc_format_card_id_string(ivar); if (bootverbose || mmc_debug) @@ -1529,7 +1711,7 @@ mmc_discover_cards(struct mmc_softc *sc) static void mmc_rescan_cards(struct mmc_softc *sc) { - struct mmc_ivars *ivar = NULL; + struct mmc_ivars *ivar; device_t *devlist; int err, i, devcount; @@ -1537,9 +1719,10 @@ mmc_rescan_cards(struct mmc_softc *sc) return; for (i = 0; i < devcount; i++) { ivar = device_get_ivars(devlist[i]); - if (mmc_select_card(sc, ivar->rca)) { + if (mmc_select_card(sc, ivar->rca) != MMC_ERR_NONE) { if (bootverbose || mmc_debug) - device_printf(sc->dev, "Card at relative address %d lost.\n", + device_printf(sc->dev, + "Card at relative address %d lost.\n", ivar->rca); device_delete_child(sc->dev, devlist[i]); free(ivar, M_DEVBUF); @@ -1561,7 +1744,8 @@ mmc_delete_cards(struct mmc_softc *sc) for (i = 0; i < devcount; i++) { ivar = device_get_ivars(devlist[i]); if (bootverbose || mmc_debug) - device_printf(sc->dev, "Card at relative address %d deleted.\n", + device_printf(sc->dev, + "Card at relative address %d deleted.\n", ivar->rca); device_delete_child(sc->dev, devlist[i]); free(ivar, M_DEVBUF); @@ -1591,7 +1775,8 @@ mmc_go_discovery(struct mmc_softc *sc) mmc_idle_cards(sc); err = mmc_send_if_cond(sc, 1); if ((bootverbose || mmc_debug) && err == 0) - device_printf(sc->dev, "SD 2.0 interface conditions: OK\n"); + device_printf(sc->dev, + "SD 2.0 interface conditions: OK\n"); if (mmc_send_app_op_cond(sc, 0, &ocr) != MMC_ERR_NONE) { if (bootverbose || mmc_debug) device_printf(sc->dev, "SD probe: failed\n"); @@ -1601,13 +1786,15 @@ mmc_go_discovery(struct mmc_softc *sc) mmcbr_set_mode(dev, mode_mmc); if (mmc_send_op_cond(sc, 0, &ocr) != MMC_ERR_NONE) { if (bootverbose || mmc_debug) - device_printf(sc->dev, "MMC probe: failed\n"); + device_printf(sc->dev, + "MMC probe: failed\n"); ocr = 0; /* Failed both, powerdown. */ } else if (bootverbose || mmc_debug) device_printf(sc->dev, "MMC probe: OK (OCR: 0x%08x)\n", ocr); } else if (bootverbose || mmc_debug) - device_printf(sc->dev, "SD probe: OK (OCR: 0x%08x)\n", ocr); + device_printf(sc->dev, "SD probe: OK (OCR: 0x%08x)\n", + ocr); sc->squelched--; mmcbr_set_ocr(dev, mmc_select_vdd(sc, ocr)); @@ -1615,7 +1802,7 @@ mmc_go_discovery(struct mmc_softc *sc) mmc_idle_cards(sc); } else { mmcbr_set_bus_mode(dev, opendrain); - mmcbr_set_clock(dev, CARD_ID_FREQUENCY); + mmcbr_set_clock(dev, SD_MMC_CARD_ID_FREQUENCY); mmcbr_update_ios(dev); /* XXX recompute vdd based on new cards? */ } @@ -1624,7 +1811,8 @@ mmc_go_discovery(struct mmc_softc *sc) * one card on the bus. */ if (bootverbose || mmc_debug) - device_printf(sc->dev, "Current OCR: 0x%08x\n", mmcbr_get_ocr(dev)); + device_printf(sc->dev, "Current OCR: 0x%08x\n", + mmcbr_get_ocr(dev)); if (mmcbr_get_ocr(dev) == 0) { device_printf(sc->dev, "No compatible cards found on bus\n"); mmc_delete_cards(sc); @@ -1646,56 +1834,69 @@ mmc_go_discovery(struct mmc_softc *sc) mmcbr_set_bus_mode(dev, pushpull); mmcbr_update_ios(dev); mmc_calculate_clock(sc); - bus_generic_attach(dev); -/* mmc_update_children_sysctl(dev);*/ } static int mmc_calculate_clock(struct mmc_softc *sc) { - int max_dtr, max_hs_dtr, max_timing; - int nkid, i, f_max; device_t *kids; struct mmc_ivars *ivar; - - f_max = mmcbr_get_f_max(sc->dev); - max_dtr = max_hs_dtr = f_max; - if ((mmcbr_get_caps(sc->dev) & MMC_CAP_HSPEED)) + int host_caps, i, nkid; + uint32_t dtr, max_dtr; + enum mmc_bus_timing max_timing, timing; + bool changed; + + max_dtr = mmcbr_get_f_max(sc->dev); + host_caps = mmcbr_get_caps(sc->dev); + if ((host_caps & MMC_CAP_MMC_DDR52) != 0) + max_timing = bus_timing_mmc_ddr52; + else if ((host_caps & MMC_CAP_HSPEED) != 0) max_timing = bus_timing_hs; else max_timing = bus_timing_normal; if (device_get_children(sc->dev, &kids, &nkid) != 0) panic("can't get children"); - for (i = 0; i < nkid; i++) { - ivar = device_get_ivars(kids[i]); - if (ivar->timing < max_timing) - max_timing = ivar->timing; - if (ivar->tran_speed < max_dtr) - max_dtr = ivar->tran_speed; - if (ivar->hs_tran_speed < max_hs_dtr) - max_hs_dtr = ivar->hs_tran_speed; + do { + changed = false; + for (i = 0; i < nkid; i++) { + ivar = device_get_ivars(kids[i]); + if (isclr(&ivar->timings, max_timing)) { + for (timing = max_timing; timing >= + bus_timing_normal; timing--) { + if (isset(&ivar->timings, timing)) { + max_timing = timing; + break; + } + } + changed = true; + } + dtr = mmc_timing_to_dtr(ivar, max_timing); + if (dtr < max_dtr) { + max_dtr = dtr; + changed = true; + } + } + } while (changed == true); + if (bootverbose || mmc_debug) { + device_printf(sc->dev, + "setting transfer rate to %d.%03dMHz (%s timing)\n", + max_dtr / 1000000, (max_dtr / 1000) % 1000, + mmc_timing_to_string(max_timing)); } for (i = 0; i < nkid; i++) { ivar = device_get_ivars(kids[i]); - if (ivar->timing == bus_timing_normal) + if ((ivar->timings & ~(1 << bus_timing_normal)) == 0) continue; - mmc_select_card(sc, ivar->rca); - mmc_set_timing(sc, max_timing); + if (mmc_select_card(sc, ivar->rca) != MMC_ERR_NONE || + mmc_set_timing(sc, ivar, max_timing) != MMC_ERR_NONE) + device_printf(sc->dev, "Card at relative address %d " + "failed to set timing.\n", ivar->rca); } mmc_select_card(sc, 0); free(kids, M_TEMP); - if (max_timing == bus_timing_hs) - max_dtr = max_hs_dtr; - if (bootverbose || mmc_debug) { - device_printf(sc->dev, - "setting transfer rate to %d.%03dMHz%s\n", - max_dtr / 1000000, (max_dtr / 1000) % 1000, - max_timing == bus_timing_hs ? " (high speed timing)" : ""); - } - mmcbr_set_timing(sc->dev, max_timing); mmcbr_set_clock(sc->dev, max_dtr); mmcbr_update_ios(sc->dev); - return max_dtr; + return (max_dtr); } static void @@ -1706,6 +1907,8 @@ mmc_scan(struct mmc_softc *sc) mmc_acquire_bus(dev, dev); mmc_go_discovery(sc); mmc_release_bus(dev, dev); + + bus_generic_attach(dev); } static int @@ -1716,6 +1919,9 @@ mmc_read_ivar(device_t bus, device_t child, int which, uintptr_t *result) switch (which) { default: return (EINVAL); + case MMC_IVAR_SPEC_VERS: + *result = ivar->csd.spec_vers; + break; case MMC_IVAR_DSR_IMP: *result = ivar->csd.dsr_imp; break; @@ -1762,6 +1968,7 @@ mmc_read_ivar(device_t bus, device_t child, int which, uintptr_t *result) static int mmc_write_ivar(device_t bus, device_t child, int which, uintptr_t value) { + /* * None are writable ATM */ @@ -1814,4 +2021,4 @@ driver_t mmc_driver = { }; devclass_t mmc_devclass; -MODULE_VERSION(mmc, 1); +MODULE_VERSION(mmc, MMC_VERSION); diff --git a/freebsd/sys/dev/mmc/mmc_ioctl.h b/freebsd/sys/dev/mmc/mmc_ioctl.h new file mode 100644 index 00000000..97cff068 --- /dev/null +++ b/freebsd/sys/dev/mmc/mmc_ioctl.h @@ -0,0 +1,64 @@ +/*- + * Copyright (c) 2017 Marius Strobl <marius@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _DEV_MMC_MMC_IOCTL_H_ +#define _DEV_MMC_MMC_IOCTL_H_ + +struct mmc_ioc_cmd { + int write_flag; /* 0: RD, 1: WR, (1 << 31): reliable WR */ + int is_acmd; /* 0: normal, 1: use CMD55 */ + uint32_t opcode; + uint32_t arg; + uint32_t response[4]; + u_int flags; + u_int blksz; + u_int blocks; + u_int __spare[4]; + uint32_t __pad; + uint64_t data_ptr; +}; + +#define mmc_ioc_cmd_set_data(mic, ptr) \ + (mic).data_ptr = (uint64_t)(uintptr_t)(ptr) + +struct mmc_ioc_multi_cmd { + uint64_t num_of_cmds; + struct mmc_ioc_cmd cmds[0]; +}; + +#define MMC_IOC_BASE 'M' + +#define MMC_IOC_CMD _IOWR(MMC_IOC_BASE, 0, struct mmc_ioc_cmd) +#define MMC_IOC_CMD_MULTI _IOWR(MMC_IOC_BASE, 1, struct mmc_ioc_multi_cmd) + +/* Maximum accepted data transfer size */ +#define MMC_IOC_MAX_BYTES (512 * 256) +/* Maximum accepted number of commands */ +#define MMC_IOC_MAX_CMDS 255 + +#endif /* _DEV_MMC_MMC_IOCTL_H_ */ diff --git a/freebsd/sys/dev/mmc/mmc_private.h b/freebsd/sys/dev/mmc/mmc_private.h new file mode 100644 index 00000000..bbca0c60 --- /dev/null +++ b/freebsd/sys/dev/mmc/mmc_private.h @@ -0,0 +1,69 @@ +/*- + * Copyright (c) 2006 Bernd Walter. All rights reserved. + * Copyright (c) 2006 M. Warner Losh. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Portions of this software may have been developed with reference to + * the SD Simplified Specification. The following disclaimer may apply: + * + * The following conditions apply to the release of the simplified + * specification ("Simplified Specification") by the SD Card Association and + * the SD Group. The Simplified Specification is a subset of the complete SD + * Specification which is owned by the SD Card Association and the SD + * Group. This Simplified Specification is provided on a non-confidential + * basis subject to the disclaimers below. Any implementation of the + * Simplified Specification may require a license from the SD Card + * Association, SD Group, SD-3C LLC or other third parties. + * + * Disclaimers: + * + * The information contained in the Simplified Specification is presented only + * as a standard specification for SD Cards and SD Host/Ancillary products and + * is provided "AS-IS" without any representations or warranties of any + * kind. No responsibility is assumed by the SD Group, SD-3C LLC or the SD + * Card Association for any damages, any infringements of patents or other + * right of the SD Group, SD-3C LLC, the SD Card Association or any third + * parties, which may result from its use. No license is granted by + * implication, estoppel or otherwise under any patent or other rights of the + * SD Group, SD-3C LLC, the SD Card Association or any third party. Nothing + * herein shall be construed as an obligation by the SD Group, the SD-3C LLC + * or the SD Card Association to disclose or distribute any technical + * information, know-how or other confidential information to any third party. + * + * $FreeBSD$ + */ + +#ifndef DEV_MMC_PRIVATE_H +#define DEV_MMC_PRIVATE_H + +struct mmc_softc { + device_t dev; + struct mtx sc_mtx; + struct intr_config_hook config_intrhook; + device_t owner; + uint32_t last_rca; + int squelched; /* suppress reporting of (expected) errors */ + int log_count; + struct timeval log_time; +}; + +#endif /* DEV_MMC_PRIVATE_H */ diff --git a/freebsd/sys/dev/mmc/mmc_subr.c b/freebsd/sys/dev/mmc/mmc_subr.c new file mode 100644 index 00000000..294fd9c0 --- /dev/null +++ b/freebsd/sys/dev/mmc/mmc_subr.c @@ -0,0 +1,254 @@ +#include <machine/rtems-bsd-kernel-space.h> + +/*- + * Copyright (c) 2006 Bernd Walter. All rights reserved. + * Copyright (c) 2006 M. Warner Losh. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Portions of this software may have been developed with reference to + * the SD Simplified Specification. The following disclaimer may apply: + * + * The following conditions apply to the release of the simplified + * specification ("Simplified Specification") by the SD Card Association and + * the SD Group. The Simplified Specification is a subset of the complete SD + * Specification which is owned by the SD Card Association and the SD + * Group. This Simplified Specification is provided on a non-confidential + * basis subject to the disclaimers below. Any implementation of the + * Simplified Specification may require a license from the SD Card + * Association, SD Group, SD-3C LLC or other third parties. + * + * Disclaimers: + * + * The information contained in the Simplified Specification is presented only + * as a standard specification for SD Cards and SD Host/Ancillary products and + * is provided "AS-IS" without any representations or warranties of any + * kind. No responsibility is assumed by the SD Group, SD-3C LLC or the SD + * Card Association for any damages, any infringements of patents or other + * right of the SD Group, SD-3C LLC, the SD Card Association or any third + * parties, which may result from its use. No license is granted by + * implication, estoppel or otherwise under any patent or other rights of the + * SD Group, SD-3C LLC, the SD Card Association or any third party. Nothing + * herein shall be construed as an obligation by the SD Group, the SD-3C LLC + * or the SD Card Association to disclose or distribute any technical + * information, know-how or other confidential information to any third party. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <rtems/bsd/sys/param.h> +#include <sys/systm.h> +#include <sys/kernel.h> +#include <rtems/bsd/sys/lock.h> +#include <sys/mutex.h> +#include <sys/time.h> + +#include <dev/mmc/bridge.h> +#include <dev/mmc/mmc_private.h> +#include <dev/mmc/mmc_subr.h> +#include <dev/mmc/mmcreg.h> +#include <dev/mmc/mmcbrvar.h> + +#include <rtems/bsd/local/mmcbus_if.h> + +#define CMD_RETRIES 3 +#define LOG_PPS 5 /* Log no more than 5 errors per second. */ + +int +mmc_wait_for_cmd(device_t brdev, device_t reqdev, struct mmc_command *cmd, + int retries) +{ + struct mmc_request mreq; + struct mmc_softc *sc; + int err; + + do { + memset(&mreq, 0, sizeof(mreq)); + memset(cmd->resp, 0, sizeof(cmd->resp)); + cmd->retries = 0; /* Retries done here, not in hardware. */ + cmd->mrq = &mreq; + if (cmd->data != NULL) + cmd->data->mrq = &mreq; + mreq.cmd = cmd; + if (MMCBUS_WAIT_FOR_REQUEST(brdev, reqdev, &mreq) != 0) + err = MMC_ERR_FAILED; + else + err = cmd->error; + } while (err != MMC_ERR_NONE && retries-- > 0); + + if (err != MMC_ERR_NONE && brdev == reqdev) { + sc = device_get_softc(brdev); + if (sc->squelched == 0 && ppsratecheck(&sc->log_time, + &sc->log_count, LOG_PPS)) { + device_printf(sc->dev, "CMD%d failed, RESULT: %d\n", + cmd->opcode, err); + } + } + + return (err); +} + +int +mmc_wait_for_app_cmd(device_t brdev, device_t reqdev, uint16_t rca, + struct mmc_command *cmd, int retries) +{ + struct mmc_command appcmd; + struct mmc_softc *sc; + int err; + + sc = device_get_softc(brdev); + + /* Squelch error reporting at lower levels, we report below. */ + sc->squelched++; + do { + memset(&appcmd, 0, sizeof(appcmd)); + appcmd.opcode = MMC_APP_CMD; + appcmd.arg = (uint32_t)rca << 16; + appcmd.flags = MMC_RSP_R1 | MMC_CMD_AC; + if (mmc_wait_for_cmd(brdev, reqdev, &appcmd, 0) != 0) + err = MMC_ERR_FAILED; + else + err = appcmd.error; + if (err == MMC_ERR_NONE) { + if (!(appcmd.resp[0] & R1_APP_CMD)) + err = MMC_ERR_FAILED; + else if (mmc_wait_for_cmd(brdev, reqdev, cmd, 0) != 0) + err = MMC_ERR_FAILED; + else + err = cmd->error; + } + } while (err != MMC_ERR_NONE && retries-- > 0); + sc->squelched--; + + if (err != MMC_ERR_NONE && brdev == reqdev) { + sc = device_get_softc(brdev); + if (sc->squelched == 0 && ppsratecheck(&sc->log_time, + &sc->log_count, LOG_PPS)) { + device_printf(sc->dev, "ACMD%d failed, RESULT: %d\n", + cmd->opcode, err); + } + } + + return (err); +} + +int +mmc_switch(device_t brdev, device_t reqdev, uint16_t rca, uint8_t set, + uint8_t index, uint8_t value, u_int timeout, bool status) +{ + struct mmc_command cmd; + int err; + + KASSERT(timeout != 0, ("%s: no timeout", __func__)); + + memset(&cmd, 0, sizeof(cmd)); + cmd.opcode = MMC_SWITCH_FUNC; + cmd.arg = (MMC_SWITCH_FUNC_WR << 24) | (index << 16) | (value << 8) | + set; + /* + * If the hardware supports busy detection but the switch timeout + * exceeds the maximum host timeout, use a R1 instead of a R1B + * response in order to keep the hardware from timing out. + */ + if (mmcbr_get_caps(brdev) & MMC_CAP_WAIT_WHILE_BUSY && + timeout > mmcbr_get_max_busy_timeout(brdev)) + cmd.flags = MMC_RSP_R1 | MMC_CMD_AC; + else + cmd.flags = MMC_RSP_R1B | MMC_CMD_AC; + err = mmc_wait_for_cmd(brdev, reqdev, &cmd, CMD_RETRIES); + if (err != MMC_ERR_NONE || status == false) + return (err); + return (mmc_switch_status(brdev, reqdev, rca, timeout)); +} + +int +mmc_switch_status(device_t brdev, device_t reqdev, uint16_t rca, u_int timeout) +{ + struct timeval cur, end; + int err; + uint32_t status; + + KASSERT(timeout != 0, ("%s: no timeout", __func__)); + + /* + * Note that when using a R1B response in mmc_switch(), bridges of + * type MMC_CAP_WAIT_WHILE_BUSY will issue mmc_send_status() only + * once and then exit the loop. + */ + for (;;) { + err = mmc_send_status(brdev, reqdev, rca, &status); + if (err != MMC_ERR_NONE) + break; + if (R1_CURRENT_STATE(status) == R1_STATE_TRAN) + break; + getmicrouptime(&cur); + if (end.tv_sec == 0 && end.tv_usec == 0) { + end.tv_usec = timeout; + timevaladd(&end, &cur); + } + if (timevalcmp(&cur, &end, >)) { + err = MMC_ERR_TIMEOUT; + break; + } + } + if (err == MMC_ERR_NONE && R1_CURRENT_STATE(status) == R1_SWITCH_ERROR) + return (MMC_ERR_FAILED); + return (err); +} + +int +mmc_send_ext_csd(device_t brdev, device_t reqdev, uint8_t *rawextcsd) +{ + struct mmc_command cmd; + struct mmc_data data; + int err; + + memset(&cmd, 0, sizeof(cmd)); + memset(&data, 0, sizeof(data)); + + memset(rawextcsd, 0, MMC_EXTCSD_SIZE); + cmd.opcode = MMC_SEND_EXT_CSD; + cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC; + cmd.data = &data; + + data.data = rawextcsd; + data.len = MMC_EXTCSD_SIZE; + data.flags = MMC_DATA_READ; + + err = mmc_wait_for_cmd(brdev, reqdev, &cmd, CMD_RETRIES); + return (err); +} + +int +mmc_send_status(device_t brdev, device_t reqdev, uint16_t rca, uint32_t *status) +{ + struct mmc_command cmd; + int err; + + memset(&cmd, 0, sizeof(cmd)); + cmd.opcode = MMC_SEND_STATUS; + cmd.arg = (uint32_t)rca << 16; + cmd.flags = MMC_RSP_R1 | MMC_CMD_AC; + err = mmc_wait_for_cmd(brdev, reqdev, &cmd, CMD_RETRIES); + *status = cmd.resp[0]; + return (err); +} diff --git a/freebsd/sys/dev/mmc/mmc_subr.h b/freebsd/sys/dev/mmc/mmc_subr.h new file mode 100644 index 00000000..6e300d2f --- /dev/null +++ b/freebsd/sys/dev/mmc/mmc_subr.h @@ -0,0 +1,72 @@ +/*- + * Copyright (c) 2006 Bernd Walter. All rights reserved. + * Copyright (c) 2006 M. Warner Losh. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Portions of this software may have been developed with reference to + * the SD Simplified Specification. The following disclaimer may apply: + * + * The following conditions apply to the release of the simplified + * specification ("Simplified Specification") by the SD Card Association and + * the SD Group. The Simplified Specification is a subset of the complete SD + * Specification which is owned by the SD Card Association and the SD + * Group. This Simplified Specification is provided on a non-confidential + * basis subject to the disclaimers below. Any implementation of the + * Simplified Specification may require a license from the SD Card + * Association, SD Group, SD-3C LLC or other third parties. + * + * Disclaimers: + * + * The information contained in the Simplified Specification is presented only + * as a standard specification for SD Cards and SD Host/Ancillary products and + * is provided "AS-IS" without any representations or warranties of any + * kind. No responsibility is assumed by the SD Group, SD-3C LLC or the SD + * Card Association for any damages, any infringements of patents or other + * right of the SD Group, SD-3C LLC, the SD Card Association or any third + * parties, which may result from its use. No license is granted by + * implication, estoppel or otherwise under any patent or other rights of the + * SD Group, SD-3C LLC, the SD Card Association or any third party. Nothing + * herein shall be construed as an obligation by the SD Group, the SD-3C LLC + * or the SD Card Association to disclose or distribute any technical + * information, know-how or other confidential information to any third party. + * + * $FreeBSD$ + */ + +#ifndef DEV_MMC_SUBR_H +#define DEV_MMC_SUBR_H + +struct mmc_command; + +int mmc_send_ext_csd(device_t brdev, device_t reqdev, uint8_t *rawextcsd); +int mmc_send_status(device_t brdev, device_t reqdev, uint16_t rca, + uint32_t *status); +int mmc_switch(device_t brdev, device_t reqdev, uint16_t rca, uint8_t set, + uint8_t index, uint8_t value, u_int timeout, bool send_status); +int mmc_switch_status(device_t brdev, device_t reqdev, uint16_t rca, + u_int timeout); +int mmc_wait_for_app_cmd(device_t brdev, device_t reqdev, uint16_t rca, + struct mmc_command *cmd, int retries); +int mmc_wait_for_cmd(device_t brdev, device_t reqdev, struct mmc_command *cmd, + int retries); + +#endif /* DEV_MMC_SUBR_H */ diff --git a/freebsd/sys/dev/mmc/mmcbrvar.h b/freebsd/sys/dev/mmc/mmcbrvar.h index 1f0a5714..77c304b4 100644 --- a/freebsd/sys/dev/mmc/mmcbrvar.h +++ b/freebsd/sys/dev/mmc/mmcbrvar.h @@ -49,14 +49,14 @@ * or the SD Card Association to disclose or distribute any technical * information, know-how or other confidential information to any third party. * - * "$FreeBSD$" + * $FreeBSD$ */ #ifndef DEV_MMC_MMCBRVAR_H -#define DEV_MMC_MMCBRVAR_H +#define DEV_MMC_MMCBRVAR_H -#include <dev/mmc/bridge.h> #include <dev/mmc/mmcreg.h> + #include <rtems/bsd/local/mmcbr_if.h> enum mmcbr_device_ivars { @@ -71,15 +71,17 @@ enum mmcbr_device_ivars { MMCBR_IVAR_OCR, MMCBR_IVAR_POWER_MODE, MMCBR_IVAR_VDD, + MMCBR_IVAR_VCCQ, MMCBR_IVAR_CAPS, MMCBR_IVAR_TIMING, - MMCBR_IVAR_MAX_DATA + MMCBR_IVAR_MAX_DATA, + MMCBR_IVAR_MAX_BUSY_TIMEOUT }; /* - * Simplified accessors for pci devices + * Simplified accessors for bridge devices */ -#define MMCBR_ACCESSOR(var, ivar, type) \ +#define MMCBR_ACCESSOR(var, ivar, type) \ __BUS_ACCESSOR(mmcbr, var, MMCBR, ivar, type) MMCBR_ACCESSOR(bus_mode, BUS_MODE, int) @@ -93,19 +95,30 @@ MMCBR_ACCESSOR(mode, MODE, int) MMCBR_ACCESSOR(ocr, OCR, int) MMCBR_ACCESSOR(power_mode, POWER_MODE, int) MMCBR_ACCESSOR(vdd, VDD, int) +MMCBR_ACCESSOR(vccq, VCCQ, int) MMCBR_ACCESSOR(caps, CAPS, int) MMCBR_ACCESSOR(timing, TIMING, int) MMCBR_ACCESSOR(max_data, MAX_DATA, int) +MMCBR_ACCESSOR(max_busy_timeout, MAX_BUSY_TIMEOUT, u_int) static int __inline mmcbr_update_ios(device_t dev) { + return (MMCBR_UPDATE_IOS(device_get_parent(dev), dev)); } static int __inline +mmcbr_switch_vccq(device_t dev) +{ + + return (MMCBR_SWITCH_VCCQ(device_get_parent(dev), dev)); +} + +static int __inline mmcbr_get_ro(device_t dev) { + return (MMCBR_GET_RO(device_get_parent(dev), dev)); } diff --git a/freebsd/sys/dev/mmc/mmcreg.h b/freebsd/sys/dev/mmc/mmcreg.h index ba4ca93a..359f31d5 100644 --- a/freebsd/sys/dev/mmc/mmcreg.h +++ b/freebsd/sys/dev/mmc/mmcreg.h @@ -1,5 +1,6 @@ /*- * Copyright (c) 2006 M. Warner Losh. All rights reserved. + * Copyright (c) 2017 Marius Strobl <marius@FreeBSD.org> * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -55,7 +56,7 @@ #define DEV_MMC_MMCREG_H /* - * This file contains the register definitions for the mmc and sd busses. + * This file contains the register definitions for the mmc and sd buses. * They are taken from publicly available sources. */ @@ -100,7 +101,7 @@ struct mmc_command { #define MMC_ERR_FAILED 4 #define MMC_ERR_INVALID 5 #define MMC_ERR_NO_MEMORY 6 -#define MMC_ERR_MAX 6 +#define MMC_ERR_MAX 6 struct mmc_data *data; /* Data segment with cmd */ struct mmc_request *mrq; /* backpointer to request */ }; @@ -140,6 +141,7 @@ struct mmc_command { #define R1_ERASE_RESET (1u << 13) /* sr, c */ #define R1_CURRENT_STATE_MASK (0xfu << 9) /* sx, b */ #define R1_READY_FOR_DATA (1u << 8) /* sx, a */ +#define R1_SWITCH_ERROR (1u << 7) /* sx, c */ #define R1_APP_CMD (1u << 5) /* sr, c */ #define R1_AKE_SEQ_ERROR (1u << 3) /* er, c */ #define R1_STATUS(x) ((x) & 0xFFFFE000) @@ -184,7 +186,7 @@ struct mmc_request { #define MMC_SET_RELATIVE_ADDR 3 #define SD_SEND_RELATIVE_ADDR 3 #define MMC_SET_DSR 4 - /* reserved: 5 */ +#define MMC_SLEEP_AWAKE 5 #define MMC_SWITCH_FUNC 6 #define MMC_SWITCH_FUNC_CMDS 0 #define MMC_SWITCH_FUNC_SET 1 @@ -207,11 +209,11 @@ struct mmc_request { #define MMC_SET_BLOCKLEN 16 #define MMC_READ_SINGLE_BLOCK 17 #define MMC_READ_MULTIPLE_BLOCK 18 - /* reserved: 19 */ +#define MMC_SEND_TUNING_BLOCK 19 +#define MMC_SEND_TUNING_BLOCK_HS200 21 /* Class 3: Stream write commands */ #define MMC_WRITE_DAT_UNTIL_STOP 20 - /* reserved: 21 */ /* reserved: 22 */ /* Class 4: Block oriented write commands */ @@ -278,7 +280,6 @@ struct mmc_request { /* reserved: 50 */ /* reserved: 57 */ - /* Application specific commands for SD */ #define ACMD_SET_BUS_WIDTH 6 #define ACMD_SD_STATUS 13 @@ -291,52 +292,153 @@ struct mmc_request { /* * EXT_CSD fields */ -#define EXT_CSD_ERASE_GRP_DEF 175 /* R/W */ -#define EXT_CSD_BUS_WIDTH 183 /* R/W */ -#define EXT_CSD_HS_TIMING 185 /* R/W */ -#define EXT_CSD_CARD_TYPE 196 /* RO */ -#define EXT_CSD_REV 192 /* RO */ -#define EXT_CSD_SEC_CNT 212 /* RO, 4 bytes */ -#define EXT_CSD_ERASE_TO_MULT 223 /* RO */ -#define EXT_CSD_ERASE_GRP_SIZE 224 /* RO */ +#define EXT_CSD_EXT_PART_ATTR 52 /* R/W, 2 bytes */ +#define EXT_CSD_ENH_START_ADDR 136 /* R/W, 4 bytes */ +#define EXT_CSD_ENH_SIZE_MULT 140 /* R/W, 3 bytes */ +#define EXT_CSD_GP_SIZE_MULT 143 /* R/W, 12 bytes */ +#define EXT_CSD_PART_SET 155 /* R/W */ +#define EXT_CSD_PART_ATTR 156 /* R/W */ +#define EXT_CSD_PART_SUPPORT 160 /* RO */ +#define EXT_CSD_RPMB_MULT 168 /* RO */ +#define EXT_CSD_BOOT_WP_STATUS 174 /* RO */ +#define EXT_CSD_ERASE_GRP_DEF 175 /* R/W */ +#define EXT_CSD_PART_CONFIG 179 /* R/W */ +#define EXT_CSD_BUS_WIDTH 183 /* R/W */ +#define EXT_CSD_STROBE_SUPPORT 184 /* RO */ +#define EXT_CSD_HS_TIMING 185 /* R/W */ +#define EXT_CSD_POWER_CLASS 187 /* R/W */ +#define EXT_CSD_CARD_TYPE 196 /* RO */ +#define EXT_CSD_DRIVER_STRENGTH 197 /* RO */ +#define EXT_CSD_REV 192 /* RO */ +#define EXT_CSD_PART_SWITCH_TO 199 /* RO */ +#define EXT_CSD_PWR_CL_52_195 200 /* RO */ +#define EXT_CSD_PWR_CL_26_195 201 /* RO */ +#define EXT_CSD_PWR_CL_52_360 202 /* RO */ +#define EXT_CSD_PWR_CL_26_360 203 /* RO */ +#define EXT_CSD_SEC_CNT 212 /* RO, 4 bytes */ +#define EXT_CSD_HC_WP_GRP_SIZE 221 /* RO */ +#define EXT_CSD_ERASE_TO_MULT 223 /* RO */ +#define EXT_CSD_ERASE_GRP_SIZE 224 /* RO */ +#define EXT_CSD_BOOT_SIZE_MULT 226 /* RO */ +#define EXT_CSD_PWR_CL_200_195 236 /* RO */ +#define EXT_CSD_PWR_CL_200_360 237 /* RO */ +#define EXT_CSD_PWR_CL_52_195_DDR 238 /* RO */ +#define EXT_CSD_PWR_CL_52_360_DDR 239 /* RO */ +#define EXT_CSD_GEN_CMD6_TIME 248 /* RO */ +#define EXT_CSD_PWR_CL_200_360_DDR 253 /* RO */ /* * EXT_CSD field definitions */ -#define EXT_CSD_CMD_SET_NORMAL 1 -#define EXT_CSD_CMD_SET_SECURE 2 -#define EXT_CSD_CMD_SET_CPSECURE 4 - -#define EXT_CSD_CARD_TYPE_26 1 -#define EXT_CSD_CARD_TYPE_52 2 - -#define EXT_CSD_BUS_WIDTH_1 0 -#define EXT_CSD_BUS_WIDTH_4 1 -#define EXT_CSD_BUS_WIDTH_8 2 - -#define MMC_TYPE_26_MAX_HS 26000000 -#define MMC_TYPE_52_MAX_HS 52000000 +#define EXT_CSD_EXT_PART_ATTR_DEFAULT 0x0 +#define EXT_CSD_EXT_PART_ATTR_SYSTEMCODE 0x1 +#define EXT_CSD_EXT_PART_ATTR_NPERSISTENT 0x2 + +#define EXT_CSD_PART_SET_COMPLETED 0x01 + +#define EXT_CSD_PART_ATTR_ENH_USR 0x01 +#define EXT_CSD_PART_ATTR_ENH_GP0 0x02 +#define EXT_CSD_PART_ATTR_ENH_GP1 0x04 +#define EXT_CSD_PART_ATTR_ENH_GP2 0x08 +#define EXT_CSD_PART_ATTR_ENH_GP3 0x10 +#define EXT_CSD_PART_ATTR_ENH_MASK 0x1f + +#define EXT_CSD_PART_SUPPORT_EN 0x01 +#define EXT_CSD_PART_SUPPORT_ENH_ATTR_EN 0x02 +#define EXT_CSD_PART_SUPPORT_EXT_ATTR_EN 0x04 + +#define EXT_CSD_BOOT_WP_STATUS_BOOT0_PWR 0x01 +#define EXT_CSD_BOOT_WP_STATUS_BOOT0_PERM 0x02 +#define EXT_CSD_BOOT_WP_STATUS_BOOT0_MASK 0x03 +#define EXT_CSD_BOOT_WP_STATUS_BOOT1_PWR 0x04 +#define EXT_CSD_BOOT_WP_STATUS_BOOT1_PERM 0x08 +#define EXT_CSD_BOOT_WP_STATUS_BOOT1_MASK 0x0c + +#define EXT_CSD_ERASE_GRP_DEF_EN 0x01 + +#define EXT_CSD_PART_CONFIG_ACC_DEFAULT 0x00 +#define EXT_CSD_PART_CONFIG_ACC_BOOT0 0x01 +#define EXT_CSD_PART_CONFIG_ACC_BOOT1 0x02 +#define EXT_CSD_PART_CONFIG_ACC_RPMB 0x03 +#define EXT_CSD_PART_CONFIG_ACC_GP0 0x04 +#define EXT_CSD_PART_CONFIG_ACC_GP1 0x05 +#define EXT_CSD_PART_CONFIG_ACC_GP2 0x06 +#define EXT_CSD_PART_CONFIG_ACC_GP3 0x07 +#define EXT_CSD_PART_CONFIG_ACC_MASK 0x07 +#define EXT_CSD_PART_CONFIG_BOOT0 0x08 +#define EXT_CSD_PART_CONFIG_BOOT1 0x10 +#define EXT_CSD_PART_CONFIG_BOOT_USR 0x38 +#define EXT_CSD_PART_CONFIG_BOOT_MASK 0x38 +#define EXT_CSD_PART_CONFIG_BOOT_ACK 0x40 + +#define EXT_CSD_CMD_SET_NORMAL 1 +#define EXT_CSD_CMD_SET_SECURE 2 +#define EXT_CSD_CMD_SET_CPSECURE 4 + +#define EXT_CSD_HS_TIMING_BC 0 +#define EXT_CSD_HS_TIMING_HS 1 +#define EXT_CSD_HS_TIMING_DDR200 2 +#define EXT_CSD_HS_TIMING_DDR400 3 +#define EXT_CSD_HS_TIMING_DRV_STR_SHIFT 4 + +#define EXT_CSD_POWER_CLASS_8BIT_MASK 0xf0 +#define EXT_CSD_POWER_CLASS_8BIT_SHIFT 4 +#define EXT_CSD_POWER_CLASS_4BIT_MASK 0x0f +#define EXT_CSD_POWER_CLASS_4BIT_SHIFT 0 + +#define EXT_CSD_CARD_TYPE_HS_26 0x0001 +#define EXT_CSD_CARD_TYPE_HS_52 0x0002 +#define EXT_CSD_CARD_TYPE_DDR_52_1_8V 0x0004 +#define EXT_CSD_CARD_TYPE_DDR_52_1_2V 0x0008 +#define EXT_CSD_CARD_TYPE_HS200_1_8V 0x0010 +#define EXT_CSD_CARD_TYPE_HS200_1_2V 0x0020 +#define EXT_CSD_CARD_TYPE_HS400_1_8V 0x0040 +#define EXT_CSD_CARD_TYPE_HS400_1_2V 0x0080 +#define EXT_CSD_CARD_TYPE_HS400ES 0x0100 + +#define EXT_CSD_BUS_WIDTH_1 0 +#define EXT_CSD_BUS_WIDTH_4 1 +#define EXT_CSD_BUS_WIDTH_8 2 +#define EXT_CSD_BUS_WIDTH_4_DDR 5 +#define EXT_CSD_BUS_WIDTH_8_DDR 6 +#define EXT_CSD_BUS_WIDTH_ES 0x80 + +#define MMC_TYPE_HS_26_MAX 26000000 +#define MMC_TYPE_HS_52_MAX 52000000 +#define MMC_TYPE_DDR52_MAX 52000000 +#define MMC_TYPE_HS200_HS400ES_MAX 200000000 /* * SD bus widths */ -#define SD_BUS_WIDTH_1 0 -#define SD_BUS_WIDTH_4 2 +#define SD_BUS_WIDTH_1 0 +#define SD_BUS_WIDTH_4 2 /* * SD Switch */ -#define SD_SWITCH_MODE_CHECK 0 -#define SD_SWITCH_MODE_SET 1 -#define SD_SWITCH_GROUP1 0 -#define SD_SWITCH_NORMAL_MODE 0 -#define SD_SWITCH_HS_MODE 1 -#define SD_SWITCH_NOCHANGE 0xF +#define SD_SWITCH_MODE_CHECK 0 +#define SD_SWITCH_MODE_SET 1 +#define SD_SWITCH_GROUP1 0 +#define SD_SWITCH_NORMAL_MODE 0 +#define SD_SWITCH_HS_MODE 1 +#define SD_SWITCH_SDR50_MODE 2 +#define SD_SWITCH_SDR104_MODE 3 +#define SD_SWITCH_DDR50 4 +#define SD_SWITCH_NOCHANGE 0xF #define SD_CLR_CARD_DETECT 0 #define SD_SET_CARD_DETECT 1 -#define SD_MAX_HS 50000000 +#define SD_HS_MAX 50000000 +#define SD_DDR50_MAX 50000000 +#define SD_SDR12_MAX 25000000 +#define SD_SDR25_MAX 50000000 +#define SD_SDR50_MAX 100000000 +#define SD_SDR104_MAX 208000000 + +/* Specifications require 400 kHz max. during ID phase. */ +#define SD_MMC_CARD_ID_FREQUENCY 400000 /* OCR bits */ @@ -373,6 +475,12 @@ struct mmc_request { #define MMC_OCR_340_350 (1U << 22) /* Vdd voltage 3.40 ~ 3.50 */ #define MMC_OCR_350_360 (1U << 23) /* Vdd voltage 3.50 ~ 3.60 */ #define MMC_OCR_MAX_VOLTAGE_SHIFT 23 +#define MMC_OCR_S18R (1U << 24) /* Switching to 1.8 V requested (SD) */ +#define MMC_OCR_S18A MMC_OCR_S18R /* Switching to 1.8 V accepted (SD) */ +#define MMC_OCR_XPC (1U << 28) /* SDXC Power Control */ +#define MMC_OCR_ACCESS_MODE_BYTE (0U << 29) /* Access Mode Byte (MMC) */ +#define MMC_OCR_ACCESS_MODE_SECT (1U << 29) /* Access Mode Sector (MMC) */ +#define MMC_OCR_ACCESS_MODE_MASK (3U << 29) #define MMC_OCR_CCS (1u << 30) /* Card Capacity status (SD vs SDHC) */ #define MMC_OCR_CARD_BUSY (1U << 31) /* Card Power up status */ @@ -419,8 +527,8 @@ struct mmc_scr { unsigned char sda_vsn; unsigned char bus_widths; -#define SD_SCR_BUS_WIDTH_1 (1<<0) -#define SD_SCR_BUS_WIDTH_4 (1<<2) +#define SD_SCR_BUS_WIDTH_1 (1 << 0) +#define SD_SCR_BUS_WIDTH_4 (1 << 2) }; struct mmc_sd_status @@ -438,11 +546,21 @@ struct mmc_sd_status }; /* + * Various MMC/SD constants + */ +#define MMC_BOOT_RPMB_BLOCK_SIZE (128 * 1024) + +#define MMC_EXTCSD_SIZE 512 + +#define MMC_PART_GP_MAX 4 +#define MMC_PART_MAX 8 + +/* * Older versions of the MMC standard had a variable sector size. However, * I've been able to find no old MMC or SD cards that have a non 512 * byte sector size anywhere, so we assume that such cards are very rare * and only note their existence in passing here... */ -#define MMC_SECTOR_SIZE 512 +#define MMC_SECTOR_SIZE 512 #endif /* DEV_MMCREG_H */ diff --git a/freebsd/sys/dev/mmc/mmcsd.c b/freebsd/sys/dev/mmc/mmcsd.c index a39d51fe..c1cfbe8b 100644 --- a/freebsd/sys/dev/mmc/mmcsd.c +++ b/freebsd/sys/dev/mmc/mmcsd.c @@ -3,6 +3,7 @@ /*- * Copyright (c) 2006 Bernd Walter. All rights reserved. * Copyright (c) 2006 M. Warner Losh. All rights reserved. + * Copyright (c) 2017 Marius Strobl <marius@FreeBSD.org> * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -60,16 +61,23 @@ __FBSDID("$FreeBSD$"); #include <sys/bio.h> #include <sys/bus.h> #include <sys/conf.h> +#include <sys/fcntl.h> +#include <sys/ioccom.h> #include <sys/kernel.h> #include <sys/kthread.h> #include <rtems/bsd/sys/lock.h> #include <sys/malloc.h> #include <sys/module.h> #include <sys/mutex.h> +#include <sys/slicer.h> #include <sys/time.h> + #include <geom/geom.h> #include <geom/geom_disk.h> +#include <dev/mmc/bridge.h> +#include <dev/mmc/mmc_ioctl.h> +#include <dev/mmc/mmc_subr.h> #include <dev/mmc/mmcbrvar.h> #include <dev/mmc/mmcreg.h> #include <dev/mmc/mmcvar.h> @@ -88,19 +96,48 @@ __FBSDID("$FreeBSD$"); #define kproc_exit kthread_exit #endif -struct mmcsd_softc { - device_t dev; - struct mtx sc_mtx; +#define MMCSD_CMD_RETRIES 5 + +#define MMCSD_FMT_BOOT "mmcsd%dboot" +#define MMCSD_FMT_GP "mmcsd%dgp" +#define MMCSD_FMT_RPMB "mmcsd%drpmb" +#define MMCSD_LABEL_ENH "enh" + +#define MMCSD_PART_NAMELEN (16 + 1) + +struct mmcsd_softc; + +struct mmcsd_part { + struct mtx part_mtx; + struct mmcsd_softc *sc; #ifndef __rtems__ struct disk *disk; struct proc *p; struct bio_queue_head bio_queue; daddr_t eblock, eend; /* Range remaining after the last erase. */ +#endif /* __rtems__ */ + u_int cnt; + u_int type; int running; int suspend; + bool ro; + char name[MMCSD_PART_NAMELEN]; +}; + +struct mmcsd_softc { + device_t dev; + device_t mmcbr; + struct mmcsd_part *part[MMC_PART_MAX]; + enum mmc_card_mode mode; + uint8_t part_curr; /* Partition currently switched to */ + uint8_t ext_csd[MMC_EXTCSD_SIZE]; + uint16_t rca; + uint32_t part_time; /* Partition switch timeout [us] */ + off_t enh_base; /* Enhanced user data area slice base ... */ + off_t enh_size; /* ... and size [bytes] */ int log_count; struct timeval log_time; -#endif /* __rtems__ */ + struct cdev *rpmb_dev; }; #ifndef __rtems__ @@ -127,26 +164,50 @@ static int mmcsd_probe(device_t dev); /* disk routines */ static int mmcsd_close(struct disk *dp); static int mmcsd_dump(void *arg, void *virtual, vm_offset_t physical, - off_t offset, size_t length); + off_t offset, size_t length); +static int mmcsd_getattr(struct bio *); +static int mmcsd_ioctl_disk(struct disk *disk, u_long cmd, void *data, + int fflag, struct thread *td); static int mmcsd_open(struct disk *dp); static void mmcsd_strategy(struct bio *bp); static void mmcsd_task(void *arg); #endif /* __rtems__ */ +/* RMPB cdev interface */ +static int mmcsd_ioctl_rpmb(struct cdev *dev, u_long cmd, caddr_t data, + int fflag, struct thread *td); + +static void mmcsd_add_part(struct mmcsd_softc *sc, u_int type, + const char *name, u_int cnt, off_t media_size, off_t erase_size, bool ro); static int mmcsd_bus_bit_width(device_t dev); #ifndef __rtems__ -static daddr_t mmcsd_delete(struct mmcsd_softc *sc, struct bio *bp); -static daddr_t mmcsd_rw(struct mmcsd_softc *sc, struct bio *bp); +static daddr_t mmcsd_delete(struct mmcsd_part *part, struct bio *bp); #endif /* __rtems__ */ - -#define MMCSD_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) -#define MMCSD_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx) -#define MMCSD_LOCK_INIT(_sc) \ - mtx_init(&_sc->sc_mtx, device_get_nameunit(_sc->dev), \ - "mmcsd", MTX_DEF) -#define MMCSD_LOCK_DESTROY(_sc) mtx_destroy(&_sc->sc_mtx); -#define MMCSD_ASSERT_LOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_OWNED); -#define MMCSD_ASSERT_UNLOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_NOTOWNED); +static int mmcsd_ioctl(struct mmcsd_part *part, u_long cmd, void *data, + int fflag); +static int mmcsd_ioctl_cmd(struct mmcsd_part *part, struct mmc_ioc_cmd *mic, + int fflag); +static uintmax_t mmcsd_pretty_size(off_t size, char *unit); +#ifndef __rtems__ +static daddr_t mmcsd_rw(struct mmcsd_part *part, struct bio *bp); +#endif /* __rtems__ */ +static int mmcsd_set_blockcount(struct mmcsd_softc *sc, u_int count, bool rel); +#ifndef __rtems__ +static int mmcsd_slicer(device_t dev, const char *provider, + struct flash_slice *slices, int *nslices); +#endif /* __rtems__ */ +static int mmcsd_switch_part(device_t bus, device_t dev, uint16_t rca, + u_int part); + +#define MMCSD_PART_LOCK(_part) mtx_lock(&(_part)->part_mtx) +#define MMCSD_PART_UNLOCK(_part) mtx_unlock(&(_part)->part_mtx) +#define MMCSD_PART_LOCK_INIT(_part) \ + mtx_init(&(_part)->part_mtx, (_part)->name, "mmcsd part", MTX_DEF) +#define MMCSD_PART_LOCK_DESTROY(_part) mtx_destroy(&(_part)->part_mtx); +#define MMCSD_PART_ASSERT_LOCKED(_part) \ + mtx_assert(&(_part)->part_mtx, MA_OWNED); +#define MMCSD_PART_ASSERT_UNLOCKED(_part) \ + mtx_assert(&(_part)->part_mtx, MA_NOTOWNED); static int mmcsd_probe(device_t dev) @@ -159,10 +220,9 @@ mmcsd_probe(device_t dev) #ifdef __rtems__ static rtems_status_code -rtems_bsd_mmcsd_set_block_size(struct mmcsd_softc *self, uint32_t block_size) +rtems_bsd_mmcsd_set_block_size(device_t dev, uint32_t block_size) { rtems_status_code status_code = RTEMS_SUCCESSFUL; - device_t dev = self->dev; struct mmc_command cmd; struct mmc_request req; @@ -183,10 +243,11 @@ rtems_bsd_mmcsd_set_block_size(struct mmcsd_softc *self, uint32_t block_size) } static int -rtems_bsd_mmcsd_disk_read_write(struct mmcsd_softc *self, rtems_blkdev_request *blkreq) +rtems_bsd_mmcsd_disk_read_write(struct mmcsd_part *part, rtems_blkdev_request *blkreq) { rtems_status_code status_code = RTEMS_SUCCESSFUL; - device_t dev = self->dev; + struct mmcsd_softc *sc = part->sc; + device_t dev = sc->dev; int shift = mmc_get_high_cap(dev) ? 0 : 9; int rca = mmc_get_rca(dev); uint32_t buffer_count = blkreq->bufnum; @@ -216,7 +277,7 @@ rtems_bsd_mmcsd_disk_read_write(struct mmcsd_softc *self, rtems_blkdev_request * data_flags = MMC_DATA_READ; } - MMCSD_LOCK(self); + MMCSD_PART_LOCK(part); for (i = 0; i < buffer_count; ++i) { rtems_blkdev_sg_buffer *sg = &blkreq->bufs [i]; @@ -293,7 +354,7 @@ rtems_bsd_mmcsd_disk_read_write(struct mmcsd_softc *self, rtems_blkdev_request * error: - MMCSD_UNLOCK(self); + MMCSD_PART_UNLOCK(part); rtems_blkdev_request_done(blkreq, status_code); @@ -303,12 +364,12 @@ error: static int rtems_bsd_mmcsd_disk_ioctl(rtems_disk_device *dd, uint32_t req, void *arg) { - struct mmcsd_softc *self = rtems_disk_get_driver_data(dd); if (req == RTEMS_BLKIO_REQUEST) { + struct mmcsd_part *part = rtems_disk_get_driver_data(dd); rtems_blkdev_request *blkreq = arg; - return rtems_bsd_mmcsd_disk_read_write(self, blkreq); + return rtems_bsd_mmcsd_disk_read_write(part, blkreq); } else if (req == RTEMS_BLKIO_CAPABILITIES) { *(uint32_t *) arg = RTEMS_BLKDEV_CAP_MULTISECTOR_CONT; return 0; @@ -321,11 +382,12 @@ static rtems_status_code rtems_bsd_mmcsd_attach_worker(rtems_media_state state, const char *src, char **dest, void *arg) { rtems_status_code status_code = RTEMS_SUCCESSFUL; - struct mmcsd_softc *self = arg; + struct mmcsd_part *part = arg; char *disk = NULL; if (state == RTEMS_MEDIA_STATE_READY) { - device_t dev = self->dev; + struct mmcsd_softc *sc = part->sc; + device_t dev = sc->dev; uint32_t block_count = mmc_get_media_size(dev); uint32_t block_size = MMC_SECTOR_SIZE; @@ -337,14 +399,14 @@ rtems_bsd_mmcsd_attach_worker(rtems_media_state state, const char *src, char **d MMCBUS_ACQUIRE_BUS(device_get_parent(dev), dev); - status_code = rtems_bsd_mmcsd_set_block_size(self, block_size); + status_code = rtems_bsd_mmcsd_set_block_size(dev, block_size); if (status_code != RTEMS_SUCCESSFUL) { printf("OOPS: set block size failed\n"); goto error; } status_code = rtems_blkdev_create(disk, block_size, - block_count, rtems_bsd_mmcsd_disk_ioctl, self); + block_count, rtems_bsd_mmcsd_disk_ioctl, part); if (status_code != RTEMS_SUCCESSFUL) { goto error; } @@ -363,128 +425,412 @@ error: static int mmcsd_attach(device_t dev) { + device_t mmcbr; struct mmcsd_softc *sc; -#ifndef __rtems__ - struct disk *d; -#else /* __rtems__ */ - struct { - char d_ident[16]; - char d_descr[64]; - } x, *d = &x; -#endif /* __rtems__ */ - intmax_t mb; - uint32_t speed; - uint32_t maxblocks; - char unit; + const uint8_t *ext_csd; + off_t erase_size, sector_size, size, wp_size; + uintmax_t bytes; + int err, i; + uint8_t rev; + bool comp, ro; + char unit[2]; sc = device_get_softc(dev); sc->dev = dev; - MMCSD_LOCK_INIT(sc); + sc->mmcbr = mmcbr = device_get_parent(dev); + sc->mode = mmcbr_get_mode(mmcbr); + sc->rca = mmc_get_rca(dev); -#ifndef __rtems__ - d = sc->disk = disk_alloc(); - d->d_open = mmcsd_open; - d->d_close = mmcsd_close; - d->d_strategy = mmcsd_strategy; - d->d_dump = mmcsd_dump; - d->d_name = "mmcsd"; - d->d_drv1 = sc; - d->d_sectorsize = mmc_get_sector_size(dev); - d->d_maxsize = mmc_get_max_data(dev) * d->d_sectorsize; - d->d_mediasize = (off_t)mmc_get_media_size(dev) * d->d_sectorsize; - d->d_stripesize = mmc_get_erase_sector(dev) * d->d_sectorsize; - d->d_unit = device_get_unit(dev); - d->d_flags = DISKFLAG_CANDELETE; - d->d_delmaxsize = mmc_get_erase_sector(dev) * d->d_sectorsize; -#endif /* __rtems__ */ - strlcpy(d->d_ident, mmc_get_card_sn_string(dev), sizeof(d->d_ident)); - strlcpy(d->d_descr, mmc_get_card_id_string(dev), sizeof(d->d_descr)); + /* Only MMC >= 4.x devices support EXT_CSD. */ + if (mmc_get_spec_vers(dev) >= 4) { + MMCBUS_ACQUIRE_BUS(mmcbr, dev); + err = mmc_send_ext_csd(mmcbr, dev, sc->ext_csd); + MMCBUS_RELEASE_BUS(mmcbr, dev); + if (err != MMC_ERR_NONE) + bzero(sc->ext_csd, sizeof(sc->ext_csd)); + } + ext_csd = sc->ext_csd; -#ifndef __rtems__ /* - * Display in most natural units. There's no cards < 1MB. The SD - * standard goes to 2GiB due to its reliance on FAT, but the data - * format supports up to 4GiB and some card makers push it up to this - * limit. The SDHC standard only goes to 32GiB due to FAT32, but the - * data format supports up to 2TiB however. 2048GB isn't too ugly, so - * we note it in passing here and don't add the code to print - * TB). Since these cards are sold in terms of MB and GB not MiB and - * GiB, report them like that. We also round to the nearest unit, since - * many cards are a few percent short, even of the power of 10 size. + * Enhanced user data area and general purpose partitions are only + * supported in revision 1.4 (EXT_CSD_REV == 4) and later, the RPMB + * partition in revision 1.5 (MMC v4.41, EXT_CSD_REV == 5) and later. */ - mb = (d->d_mediasize + 1000000 / 2 - 1) / 1000000; -#else /* __rtems__ */ - mb = mmc_get_media_size(dev); - mb *= mmc_get_sector_size(dev); - mb = (mb + 1000000 / 2 - 1) / 1000000; -#endif /* __rtems__ */ - unit = 'M'; - if (mb >= 1000) { - unit = 'G'; - mb = (mb + 1000 / 2 - 1) / 1000; + rev = ext_csd[EXT_CSD_REV]; + + /* + * Ignore user-creatable enhanced user data area and general purpose + * partitions partitions as long as partitioning hasn't been finished. + */ + comp = (ext_csd[EXT_CSD_PART_SET] & EXT_CSD_PART_SET_COMPLETED) != 0; + + /* + * Add enhanced user data area slice, unless it spans the entirety of + * the user data area. The enhanced area is of a multiple of high + * capacity write protect groups ((ERASE_GRP_SIZE + HC_WP_GRP_SIZE) * + * 512 KB) and its offset given in either sectors or bytes, depending + * on whether it's a high capacity device or not. + * NB: The slicer and its slices need to be registered before adding + * the disk for the corresponding user data area as re-tasting is + * racy. + */ + sector_size = mmc_get_sector_size(dev); + size = ext_csd[EXT_CSD_ENH_SIZE_MULT] + + (ext_csd[EXT_CSD_ENH_SIZE_MULT + 1] << 8) + + (ext_csd[EXT_CSD_ENH_SIZE_MULT + 2] << 16); + if (rev >= 4 && comp == TRUE && size > 0 && + (ext_csd[EXT_CSD_PART_SUPPORT] & + EXT_CSD_PART_SUPPORT_ENH_ATTR_EN) != 0 && + (ext_csd[EXT_CSD_PART_ATTR] & (EXT_CSD_PART_ATTR_ENH_USR)) != 0) { + erase_size = ext_csd[EXT_CSD_ERASE_GRP_SIZE] * 1024 * + MMC_SECTOR_SIZE; + wp_size = ext_csd[EXT_CSD_HC_WP_GRP_SIZE]; + size *= erase_size * wp_size; + if (size != mmc_get_media_size(dev) * sector_size) { + sc->enh_size = size; + sc->enh_base = (ext_csd[EXT_CSD_ENH_START_ADDR] + + (ext_csd[EXT_CSD_ENH_START_ADDR + 1] << 8) + + (ext_csd[EXT_CSD_ENH_START_ADDR + 2] << 16) + + (ext_csd[EXT_CSD_ENH_START_ADDR + 3] << 24)) * + (mmc_get_high_cap(dev) ? MMC_SECTOR_SIZE : 1); + } else if (bootverbose) + device_printf(dev, + "enhanced user data area spans entire device\n"); } + + /* + * Add default partition. This may be the only one or the user + * data area in case partitions are supported. + */ + ro = mmc_get_read_only(dev); + mmcsd_add_part(sc, EXT_CSD_PART_CONFIG_ACC_DEFAULT, "mmcsd", + device_get_unit(dev), mmc_get_media_size(dev) * sector_size, + mmc_get_erase_sector(dev) * sector_size, ro); + + if (mmc_get_spec_vers(dev) < 3) + return (0); + + /* Belatedly announce enhanced user data slice. */ + if (sc->enh_size != 0) { + bytes = mmcsd_pretty_size(size, unit); + printf(FLASH_SLICES_FMT ": %ju%sB enhanced user data area " + "slice offset 0x%jx at %s\n", device_get_nameunit(dev), + MMCSD_LABEL_ENH, bytes, unit, (uintmax_t)sc->enh_base, + device_get_nameunit(dev)); + } + + /* + * Determine partition switch timeout (provided in units of 10 ms) + * and ensure it's at least 300 ms as some eMMC chips lie. + */ + sc->part_time = max(ext_csd[EXT_CSD_PART_SWITCH_TO] * 10 * 1000, + 300 * 1000); + + /* Add boot partitions, which are of a fixed multiple of 128 KB. */ + size = ext_csd[EXT_CSD_BOOT_SIZE_MULT] * MMC_BOOT_RPMB_BLOCK_SIZE; + if (size > 0 && (mmcbr_get_caps(mmcbr) & MMC_CAP_BOOT_NOACC) == 0) { + mmcsd_add_part(sc, EXT_CSD_PART_CONFIG_ACC_BOOT0, + MMCSD_FMT_BOOT, 0, size, MMC_BOOT_RPMB_BLOCK_SIZE, + ro | ((ext_csd[EXT_CSD_BOOT_WP_STATUS] & + EXT_CSD_BOOT_WP_STATUS_BOOT0_MASK) != 0)); + mmcsd_add_part(sc, EXT_CSD_PART_CONFIG_ACC_BOOT1, + MMCSD_FMT_BOOT, 1, size, MMC_BOOT_RPMB_BLOCK_SIZE, + ro | ((ext_csd[EXT_CSD_BOOT_WP_STATUS] & + EXT_CSD_BOOT_WP_STATUS_BOOT1_MASK) != 0)); + } + + /* Add RPMB partition, which also is of a fixed multiple of 128 KB. */ + size = ext_csd[EXT_CSD_RPMB_MULT] * MMC_BOOT_RPMB_BLOCK_SIZE; + if (rev >= 5 && size > 0) + mmcsd_add_part(sc, EXT_CSD_PART_CONFIG_ACC_RPMB, + MMCSD_FMT_RPMB, 0, size, MMC_BOOT_RPMB_BLOCK_SIZE, ro); + + if (rev <= 3 || comp == FALSE) + return (0); + + /* + * Add general purpose partitions, which are of a multiple of high + * capacity write protect groups, too. + */ + if ((ext_csd[EXT_CSD_PART_SUPPORT] & EXT_CSD_PART_SUPPORT_EN) != 0) { + erase_size = ext_csd[EXT_CSD_ERASE_GRP_SIZE] * 1024 * + MMC_SECTOR_SIZE; + wp_size = ext_csd[EXT_CSD_HC_WP_GRP_SIZE]; + for (i = 0; i < MMC_PART_GP_MAX; i++) { + size = ext_csd[EXT_CSD_GP_SIZE_MULT + i * 3] + + (ext_csd[EXT_CSD_GP_SIZE_MULT + i * 3 + 1] << 8) + + (ext_csd[EXT_CSD_GP_SIZE_MULT + i * 3 + 2] << 16); + if (size == 0) + continue; + mmcsd_add_part(sc, EXT_CSD_PART_CONFIG_ACC_GP0 + i, + MMCSD_FMT_GP, i, size * erase_size * wp_size, + erase_size, ro); + } + } + return (0); +} + +static uintmax_t +mmcsd_pretty_size(off_t size, char *unit) +{ + uintmax_t bytes; + int i; + /* - * Report the clock speed of the underlying hardware, which might be - * different than what the card reports due to hardware limitations. - * Report how many blocks the hardware transfers at once. + * Display in most natural units. There's no card < 1MB. However, + * RPMB partitions occasionally are smaller than that, though. The + * SD standard goes to 2 GiB due to its reliance on FAT, but the data + * format supports up to 4 GiB and some card makers push it up to this + * limit. The SDHC standard only goes to 32 GiB due to FAT32, but the + * data format supports up to 2 TiB however. 2048 GB isn't too ugly, + * so we note it in passing here and don't add the code to print TB). + * Since these cards are sold in terms of MB and GB not MiB and GiB, + * report them like that. We also round to the nearest unit, since + * many cards are a few percent short, even of the power of 10 size. */ - speed = mmcbr_get_clock(device_get_parent(dev)); - maxblocks = mmc_get_max_data(dev); - device_printf(dev, "%ju%cB <%s>%s at %s %d.%01dMHz/%dbit/%d-block\n", - mb, unit, d->d_descr, - mmc_get_read_only(dev) ? " (read-only)" : "", - device_get_nameunit(device_get_parent(dev)), - speed / 1000000, (speed / 100000) % 10, - mmcsd_bus_bit_width(dev), maxblocks); + bytes = size; + unit[0] = unit[1] = '\0'; + for (i = 0; i <= 2 && bytes >= 1000; i++) { + bytes = (bytes + 1000 / 2 - 1) / 1000; + switch (i) { + case 0: + unit[0] = 'k'; + break; + case 1: + unit[0] = 'M'; + break; + case 2: + unit[0] = 'G'; + break; + default: + break; + } + } + return (bytes); +} + +static struct cdevsw mmcsd_rpmb_cdevsw = { + .d_version = D_VERSION, + .d_name = "mmcsdrpmb", + .d_ioctl = mmcsd_ioctl_rpmb +}; + +static void +mmcsd_add_part(struct mmcsd_softc *sc, u_int type, const char *name, u_int cnt, + off_t media_size, off_t erase_size, bool ro) +{ + struct make_dev_args args; + device_t dev, mmcbr; + const char *ext; + const uint8_t *ext_csd; + struct mmcsd_part *part; #ifndef __rtems__ - disk_create(d, DISK_VERSION); - bioq_init(&sc->bio_queue); - - sc->running = 1; - sc->suspend = 0; - sc->eblock = sc->eend = 0; - kproc_create(&mmcsd_task, sc, &sc->p, 0, 0, "%s: mmc/sd card", - device_get_nameunit(dev)); + struct disk *d; +#endif /* __rtems__ */ + uintmax_t bytes; + u_int gp; + uint32_t speed; + uint8_t extattr; + bool enh; + char unit[2]; + + dev = sc->dev; + mmcbr = sc->mmcbr; + part = sc->part[type] = malloc(sizeof(*part), M_DEVBUF, + M_WAITOK | M_ZERO); + part->sc = sc; + part->cnt = cnt; + part->type = type; + part->ro = ro; + snprintf(part->name, sizeof(part->name), name, device_get_unit(dev)); + + /* For the RPMB partition, allow IOCTL access only. */ + if (type == EXT_CSD_PART_CONFIG_ACC_RPMB) { + make_dev_args_init(&args); + args.mda_flags = MAKEDEV_CHECKNAME | MAKEDEV_WAITOK; + args.mda_devsw = &mmcsd_rpmb_cdevsw; + args.mda_uid = UID_ROOT; + args.mda_gid = GID_OPERATOR; + args.mda_mode = 0640; + args.mda_si_drv1 = part; + if (make_dev_s(&args, &sc->rpmb_dev, "%s", part->name) != 0) { + device_printf(dev, "Failed to make RPMB device\n"); + free(part, M_DEVBUF); + return; + } + } else { + MMCSD_PART_LOCK_INIT(part); + +#ifndef __rtems__ + d = part->disk = disk_alloc(); + d->d_open = mmcsd_open; + d->d_close = mmcsd_close; + d->d_strategy = mmcsd_strategy; + d->d_ioctl = mmcsd_ioctl_disk; + d->d_dump = mmcsd_dump; + d->d_getattr = mmcsd_getattr; + d->d_name = part->name; + d->d_drv1 = part; + d->d_sectorsize = mmc_get_sector_size(dev); + d->d_maxsize = mmc_get_max_data(dev) * d->d_sectorsize; + d->d_mediasize = media_size; + d->d_stripesize = erase_size; + d->d_unit = cnt; + d->d_flags = DISKFLAG_CANDELETE; + d->d_delmaxsize = erase_size; + strlcpy(d->d_ident, mmc_get_card_sn_string(dev), + sizeof(d->d_ident)); + strlcpy(d->d_descr, mmc_get_card_id_string(dev), + sizeof(d->d_descr)); + d->d_rotation_rate = DISK_RR_NON_ROTATING; + + disk_create(d, DISK_VERSION); + bioq_init(&part->bio_queue); + + part->running = 1; + kproc_create(&mmcsd_task, part, &part->p, 0, 0, + "%s%d: mmc/sd card", part->name, cnt); #else /* __rtems__ */ - rtems_status_code status_code = rtems_media_server_disk_attach( - device_get_name(dev), - rtems_bsd_mmcsd_attach_worker, - sc - ); - BSD_ASSERT(status_code == RTEMS_SUCCESSFUL); + rtems_status_code status_code = rtems_media_server_disk_attach( + part->name, rtems_bsd_mmcsd_attach_worker, part); + BSD_ASSERT(status_code == RTEMS_SUCCESSFUL); #endif /* __rtems__ */ + } + + bytes = mmcsd_pretty_size(media_size, unit); + if (type == EXT_CSD_PART_CONFIG_ACC_DEFAULT) { + speed = mmcbr_get_clock(mmcbr); + printf("%s%d: %ju%sB <%s>%s at %s %d.%01dMHz/%dbit/%d-block\n", + part->name, cnt, bytes, unit, mmc_get_card_id_string(dev), + ro ? " (read-only)" : "", device_get_nameunit(mmcbr), + speed / 1000000, (speed / 100000) % 10, + mmcsd_bus_bit_width(dev), mmc_get_max_data(dev)); + } else if (type == EXT_CSD_PART_CONFIG_ACC_RPMB) { + printf("%s: %ju%sB partion %d%s at %s\n", part->name, bytes, + unit, type, ro ? " (read-only)" : "", + device_get_nameunit(dev)); + } else { + enh = false; + ext = NULL; + extattr = 0; + if (type >= EXT_CSD_PART_CONFIG_ACC_GP0 && + type <= EXT_CSD_PART_CONFIG_ACC_GP3) { + ext_csd = sc->ext_csd; + gp = type - EXT_CSD_PART_CONFIG_ACC_GP0; + if ((ext_csd[EXT_CSD_PART_SUPPORT] & + EXT_CSD_PART_SUPPORT_ENH_ATTR_EN) != 0 && + (ext_csd[EXT_CSD_PART_ATTR] & + (EXT_CSD_PART_ATTR_ENH_GP0 << gp)) != 0) + enh = true; + else if ((ext_csd[EXT_CSD_PART_SUPPORT] & + EXT_CSD_PART_SUPPORT_EXT_ATTR_EN) != 0) { + extattr = (ext_csd[EXT_CSD_EXT_PART_ATTR + + (gp / 2)] >> (4 * (gp % 2))) & 0xF; + switch (extattr) { + case EXT_CSD_EXT_PART_ATTR_DEFAULT: + break; + case EXT_CSD_EXT_PART_ATTR_SYSTEMCODE: + ext = "system code"; + break; + case EXT_CSD_EXT_PART_ATTR_NPERSISTENT: + ext = "non-persistent"; + break; + default: + ext = "reserved"; + break; + } + } + } + if (ext == NULL) + printf("%s%d: %ju%sB partion %d%s%s at %s\n", + part->name, cnt, bytes, unit, type, enh ? + " enhanced" : "", ro ? " (read-only)" : "", + device_get_nameunit(dev)); + else + printf("%s%d: %ju%sB partion %d extended 0x%x " + "(%s)%s at %s\n", part->name, cnt, bytes, unit, + type, extattr, ext, ro ? " (read-only)" : "", + device_get_nameunit(dev)); + } +} +#ifndef __rtems__ +static int +mmcsd_slicer(device_t dev, const char *provider, + struct flash_slice *slices, int *nslices) +{ + char name[MMCSD_PART_NAMELEN]; + struct mmcsd_softc *sc; + struct mmcsd_part *part; + + *nslices = 0; + if (slices == NULL) + return (ENOMEM); + + sc = device_get_softc(dev); + if (sc->enh_size == 0) + return (ENXIO); + + part = sc->part[EXT_CSD_PART_CONFIG_ACC_DEFAULT]; + snprintf(name, sizeof(name), "%s%d", part->disk->d_name, + part->disk->d_unit); + if (strcmp(name, provider) != 0) + return (ENXIO); + + *nslices = 1; + slices[0].base = sc->enh_base; + slices[0].size = sc->enh_size; + slices[0].label = MMCSD_LABEL_ENH; return (0); } +#endif /* __rtems__ */ static int mmcsd_detach(device_t dev) { +#ifndef __rtems__ struct mmcsd_softc *sc = device_get_softc(dev); + struct mmcsd_part *part; + int i; + + for (i = 0; i < MMC_PART_MAX; i++) { + part = sc->part[i]; + if (part != NULL && part->disk != NULL) { + MMCSD_PART_LOCK(part); + part->suspend = 0; + if (part->running > 0) { + /* kill thread */ + part->running = 0; + wakeup(part); + /* wait for thread to finish. */ + while (part->running != -1) + msleep(part, &part->part_mtx, 0, + "detach", 0); + } + MMCSD_PART_UNLOCK(part); + } + } -#ifndef __rtems__ - MMCSD_LOCK(sc); - sc->suspend = 0; - if (sc->running > 0) { - /* kill thread */ - sc->running = 0; - wakeup(sc); - /* wait for thread to finish. */ - while (sc->running != -1) - msleep(sc, &sc->sc_mtx, 0, "detach", 0); - } - MMCSD_UNLOCK(sc); - - /* Flush the request queue. */ - bioq_flush(&sc->bio_queue, NULL, ENXIO); - /* kill disk */ - disk_destroy(sc->disk); + if (sc->rpmb_dev != NULL) + destroy_dev(sc->rpmb_dev); + + for (i = 0; i < MMC_PART_MAX; i++) { + part = sc->part[i]; + if (part != NULL) { + if (part->disk != NULL) { + /* Flush the request queue. */ + bioq_flush(&part->bio_queue, NULL, ENXIO); + /* kill disk */ + disk_destroy(part->disk); + + MMCSD_PART_LOCK_DESTROY(part); + } + free(part, M_DEVBUF); + } + } #else /* __rtems__ */ BSD_PANIC("FIXME"); #endif /* __rtems__ */ - - MMCSD_LOCK_DESTROY(sc); - return (0); } @@ -493,18 +839,26 @@ mmcsd_suspend(device_t dev) { #ifndef __rtems__ struct mmcsd_softc *sc = device_get_softc(dev); - - MMCSD_LOCK(sc); - sc->suspend = 1; - if (sc->running > 0) { - /* kill thread */ - sc->running = 0; - wakeup(sc); - /* wait for thread to finish. */ - while (sc->running != -1) - msleep(sc, &sc->sc_mtx, 0, "detach", 0); - } - MMCSD_UNLOCK(sc); + struct mmcsd_part *part; + int i; + + for (i = 0; i < MMC_PART_MAX; i++) { + part = sc->part[i]; + if (part != NULL && part->disk != NULL) { + MMCSD_PART_LOCK(part); + part->suspend = 1; + if (part->running > 0) { + /* kill thread */ + part->running = 0; + wakeup(part); + /* wait for thread to finish. */ + while (part->running != -1) + msleep(part, &part->part_mtx, 0, + "detach", 0); + } + MMCSD_PART_UNLOCK(part); + } + } #else /* __rtems__ */ BSD_PANIC("FIXME"); #endif /* __rtems__ */ @@ -516,16 +870,23 @@ mmcsd_resume(device_t dev) { #ifndef __rtems__ struct mmcsd_softc *sc = device_get_softc(dev); - - MMCSD_LOCK(sc); - sc->suspend = 0; - if (sc->running <= 0) { - sc->running = 1; - MMCSD_UNLOCK(sc); - kproc_create(&mmcsd_task, sc, &sc->p, 0, 0, "%s: mmc/sd card", - device_get_nameunit(dev)); - } else - MMCSD_UNLOCK(sc); + struct mmcsd_part *part; + int i; + + for (i = 0; i < MMC_PART_MAX; i++) { + part = sc->part[i]; + if (part != NULL && part->disk != NULL) { + MMCSD_PART_LOCK(part); + part->suspend = 0; + if (part->running <= 0) { + part->running = 1; + kproc_create(&mmcsd_task, part, &part->p, 0, 0, + "%s%d: mmc/sd card", part->name, part->cnt); + MMCSD_PART_UNLOCK(part); + } else + MMCSD_PART_UNLOCK(part); + } + } #else /* __rtems__ */ BSD_PANIC("FIXME"); #endif /* __rtems__ */ @@ -534,14 +895,14 @@ mmcsd_resume(device_t dev) #ifndef __rtems__ static int -mmcsd_open(struct disk *dp) +mmcsd_open(struct disk *dp __unused) { return (0); } static int -mmcsd_close(struct disk *dp) +mmcsd_close(struct disk *dp __unused) { return (0); @@ -551,47 +912,339 @@ static void mmcsd_strategy(struct bio *bp) { struct mmcsd_softc *sc; - - sc = (struct mmcsd_softc *)bp->bio_disk->d_drv1; - MMCSD_LOCK(sc); - if (sc->running > 0 || sc->suspend > 0) { - bioq_disksort(&sc->bio_queue, bp); - MMCSD_UNLOCK(sc); - wakeup(sc); + struct mmcsd_part *part; + + part = bp->bio_disk->d_drv1; + sc = part->sc; + MMCSD_PART_LOCK(part); + if (part->running > 0 || part->suspend > 0) { + bioq_disksort(&part->bio_queue, bp); + MMCSD_PART_UNLOCK(part); + wakeup(part); } else { - MMCSD_UNLOCK(sc); + MMCSD_PART_UNLOCK(part); biofinish(bp, NULL, ENXIO); } } +#endif /* __rtems__ */ + +static int +mmcsd_ioctl_rpmb(struct cdev *dev, u_long cmd, caddr_t data, + int fflag, struct thread *td __unused) +{ + + return (mmcsd_ioctl(dev->si_drv1, cmd, data, fflag)); +} + +#ifndef __rtems__ +static int +mmcsd_ioctl_disk(struct disk *disk, u_long cmd, void *data, int fflag, + struct thread *td __unused) +{ + + return (mmcsd_ioctl(disk->d_drv1, cmd, data, fflag)); +} +#endif /* __rtems__ */ + +static int +mmcsd_ioctl(struct mmcsd_part *part, u_long cmd, void *data, int fflag) +{ + struct mmc_ioc_cmd *mic; + struct mmc_ioc_multi_cmd *mimc; + int i, err; + u_long cnt, size; + + if ((fflag & FREAD) == 0) + return (EBADF); + + err = 0; + switch (cmd) { + case MMC_IOC_CMD: + mic = data; + err = mmcsd_ioctl_cmd(part, data, fflag); + break; + case MMC_IOC_CMD_MULTI: + mimc = data; + if (mimc->num_of_cmds == 0) + break; + if (mimc->num_of_cmds > MMC_IOC_MAX_CMDS) + return (EINVAL); + cnt = mimc->num_of_cmds; + size = sizeof(*mic) * cnt; + mic = malloc(size, M_TEMP, M_WAITOK); + err = copyin((const void *)mimc->cmds, mic, size); + if (err != 0) + break; + for (i = 0; i < cnt; i++) { + err = mmcsd_ioctl_cmd(part, &mic[i], fflag); + if (err != 0) + break; + } + free(mic, M_TEMP); + break; + default: + return (ENOIOCTL); + } + return (err); +} + +static int +mmcsd_ioctl_cmd(struct mmcsd_part *part, struct mmc_ioc_cmd *mic, int fflag) +{ + struct mmc_command cmd; + struct mmc_data data; + struct mmcsd_softc *sc; + device_t dev, mmcbr; + void *dp; + u_long len; + int err, retries; + uint32_t status; + uint16_t rca; + + if ((fflag & FWRITE) == 0 && mic->write_flag != 0) + return (EBADF); + + if (part->ro == TRUE && mic->write_flag != 0) + return (EROFS); + + err = 0; + dp = NULL; + len = mic->blksz * mic->blocks; + if (len > MMC_IOC_MAX_BYTES) + return (EOVERFLOW); + if (len != 0) { + dp = malloc(len, M_TEMP, M_WAITOK); + err = copyin((void *)(uintptr_t)mic->data_ptr, dp, len); + if (err != 0) + goto out; + } + memset(&cmd, 0, sizeof(cmd)); + memset(&data, 0, sizeof(data)); + cmd.opcode = mic->opcode; + cmd.arg = mic->arg; + cmd.flags = mic->flags; + if (len != 0) { + data.len = len; + data.data = dp; + data.flags = mic->write_flag != 0 ? MMC_DATA_WRITE : + MMC_DATA_READ; + cmd.data = &data; + } + sc = part->sc; + rca = sc->rca; + if (mic->is_acmd == 0) { + /* Enforce/patch/restrict RCA-based commands */ + switch (cmd.opcode) { + case MMC_SET_RELATIVE_ADDR: + case MMC_SELECT_CARD: + err = EPERM; + goto out; + case MMC_STOP_TRANSMISSION: + if ((cmd.arg & 0x1) == 0) + break; + /* FALLTHROUGH */ + case MMC_SLEEP_AWAKE: + case MMC_SEND_CSD: + case MMC_SEND_CID: + case MMC_SEND_STATUS: + case MMC_GO_INACTIVE_STATE: + case MMC_FAST_IO: + case MMC_APP_CMD: + cmd.arg = (cmd.arg & 0x0000FFFF) | (rca << 16); + break; + default: + break; + } + } + dev = sc->dev; + mmcbr = sc->mmcbr; + MMCBUS_ACQUIRE_BUS(mmcbr, dev); + err = mmcsd_switch_part(mmcbr, dev, rca, part->type); + if (err != MMC_ERR_NONE) + goto release; + if (part->type == EXT_CSD_PART_CONFIG_ACC_RPMB) { + err = mmcsd_set_blockcount(sc, mic->blocks, + mic->write_flag & (1 << 31)); + if (err != MMC_ERR_NONE) + goto release; + } + if (mic->is_acmd != 0) + (void)mmc_wait_for_app_cmd(mmcbr, dev, rca, &cmd, 0); + else + (void)mmc_wait_for_cmd(mmcbr, dev, &cmd, 0); + if (part->type == EXT_CSD_PART_CONFIG_ACC_RPMB) { + /* + * If the request went to the RPMB partition, try to ensure + * that the command actually has completed ... + */ + retries = MMCSD_CMD_RETRIES; + do { + err = mmc_send_status(mmcbr, dev, rca, &status); + if (err != MMC_ERR_NONE) + break; + if (R1_STATUS(status) == 0 && + R1_CURRENT_STATE(status) != R1_STATE_PRG) + break; + DELAY(1000); + } while (retries-- > 0); + + /* ... and always switch back to the default partition. */ + err = mmcsd_switch_part(mmcbr, dev, rca, + EXT_CSD_PART_CONFIG_ACC_DEFAULT); + if (err != MMC_ERR_NONE) + goto release; + } + /* + * If EXT_CSD was changed, our copy is outdated now. Specifically, + * the upper bits of EXT_CSD_PART_CONFIG used in mmcsd_switch_part(), + * so retrieve EXT_CSD again. + */ + if (cmd.opcode == MMC_SWITCH_FUNC) { + err = mmc_send_ext_csd(mmcbr, dev, sc->ext_csd); + if (err != MMC_ERR_NONE) + goto release; + } + MMCBUS_RELEASE_BUS(mmcbr, dev); + if (cmd.error != MMC_ERR_NONE) { + switch (cmd.error) { + case MMC_ERR_TIMEOUT: + err = ETIMEDOUT; + break; + case MMC_ERR_BADCRC: + err = EILSEQ; + break; + case MMC_ERR_INVALID: + err = EINVAL; + break; + case MMC_ERR_NO_MEMORY: + err = ENOMEM; + break; + default: + err = EIO; + break; + } + goto out; + } + memcpy(mic->response, cmd.resp, 4 * sizeof(uint32_t)); + if (mic->write_flag == 0 && len != 0) { + err = copyout(dp, (void *)(uintptr_t)mic->data_ptr, len); + if (err != 0) + goto out; + } + goto out; + +release: + MMCBUS_RELEASE_BUS(mmcbr, dev); + err = EIO; + +out: + if (dp != NULL) + free(dp, M_TEMP); + return (err); +} + +#ifndef __rtems__ +static int +mmcsd_getattr(struct bio *bp) +{ + struct mmcsd_part *part; + device_t dev; + + if (strcmp(bp->bio_attribute, "MMC::device") == 0) { + if (bp->bio_length != sizeof(dev)) + return (EFAULT); + part = bp->bio_disk->d_drv1; + dev = part->sc->dev; + bcopy(&dev, bp->bio_data, sizeof(dev)); + bp->bio_completed = bp->bio_length; + return (0); + } + return (-1); +} +#endif /* __rtems__ */ + +static int +mmcsd_set_blockcount(struct mmcsd_softc *sc, u_int count, bool reliable) +{ + struct mmc_command cmd; + struct mmc_request req; + + memset(&req, 0, sizeof(req)); + memset(&cmd, 0, sizeof(cmd)); + cmd.mrq = &req; + req.cmd = &cmd; + cmd.opcode = MMC_SET_BLOCK_COUNT; + cmd.arg = count & 0x0000FFFF; + if (reliable) + cmd.arg |= 1 << 31; + cmd.flags = MMC_RSP_R1 | MMC_CMD_AC; + MMCBUS_WAIT_FOR_REQUEST(sc->mmcbr, sc->dev, &req); + return (cmd.error); +} + +static int +mmcsd_switch_part(device_t bus, device_t dev, uint16_t rca, u_int part) +{ + struct mmcsd_softc *sc; + int err; + uint8_t value; + + sc = device_get_softc(dev); + + if (sc->part_curr == part) + return (MMC_ERR_NONE); + + if (sc->mode == mode_sd) + return (MMC_ERR_NONE); + + value = (sc->ext_csd[EXT_CSD_PART_CONFIG] & + ~EXT_CSD_PART_CONFIG_ACC_MASK) | part; + /* Jump! */ + err = mmc_switch(bus, dev, rca, EXT_CSD_CMD_SET_NORMAL, + EXT_CSD_PART_CONFIG, value, sc->part_time, true); + if (err != MMC_ERR_NONE) + return (err); + + sc->ext_csd[EXT_CSD_PART_CONFIG] = value; + sc->part_curr = part; + return (MMC_ERR_NONE); +} +#ifndef __rtems__ static const char * mmcsd_errmsg(int e) { + if (e < 0 || e > MMC_ERR_MAX) return "Bad error code"; return errmsg[e]; } static daddr_t -mmcsd_rw(struct mmcsd_softc *sc, struct bio *bp) +mmcsd_rw(struct mmcsd_part *part, struct bio *bp) { daddr_t block, end; struct mmc_command cmd; struct mmc_command stop; struct mmc_request req; struct mmc_data data; - device_t dev = sc->dev; - int sz = sc->disk->d_sectorsize; - device_t mmcbr = device_get_parent(dev); + struct mmcsd_softc *sc; + device_t dev, mmcbr; + int numblocks, sz; + char *vaddr; + + sc = part->sc; + dev = sc->dev; + mmcbr = sc->mmcbr; block = bp->bio_pblkno; + sz = part->disk->d_sectorsize; end = bp->bio_pblkno + (bp->bio_bcount / sz); while (block < end) { - char *vaddr = bp->bio_data + - (block - bp->bio_pblkno) * sz; - int numblocks = min(end - block, mmc_get_max_data(dev)); + vaddr = bp->bio_data + (block - bp->bio_pblkno) * sz; + numblocks = min(end - block, mmc_get_max_data(dev)); memset(&req, 0, sizeof(req)); - memset(&cmd, 0, sizeof(cmd)); + memset(&cmd, 0, sizeof(cmd)); memset(&stop, 0, sizeof(stop)); memset(&data, 0, sizeof(data)); cmd.mrq = &req; @@ -629,10 +1282,11 @@ mmcsd_rw(struct mmcsd_softc *sc, struct bio *bp) } MMCBUS_WAIT_FOR_REQUEST(mmcbr, dev, &req); if (req.cmd->error != MMC_ERR_NONE) { - if (ppsratecheck(&sc->log_time, &sc->log_count, LOG_PPS)) { + if (ppsratecheck(&sc->log_time, &sc->log_count, + LOG_PPS)) device_printf(dev, "Error indicated: %d %s\n", - req.cmd->error, mmcsd_errmsg(req.cmd->error)); - } + req.cmd->error, + mmcsd_errmsg(req.cmd->error)); break; } block += numblocks; @@ -641,33 +1295,37 @@ mmcsd_rw(struct mmcsd_softc *sc, struct bio *bp) } static daddr_t -mmcsd_delete(struct mmcsd_softc *sc, struct bio *bp) +mmcsd_delete(struct mmcsd_part *part, struct bio *bp) { daddr_t block, end, start, stop; struct mmc_command cmd; struct mmc_request req; - device_t dev = sc->dev; - int sz = sc->disk->d_sectorsize; - int erase_sector; - device_t mmcbr = device_get_parent(dev); + struct mmcsd_softc *sc; + device_t dev, mmcbr; + int erase_sector, sz; + + sc = part->sc; + dev = sc->dev; + mmcbr = sc->mmcbr; block = bp->bio_pblkno; + sz = part->disk->d_sectorsize; end = bp->bio_pblkno + (bp->bio_bcount / sz); /* Coalesce with part remaining from previous request. */ - if (block > sc->eblock && block <= sc->eend) - block = sc->eblock; - if (end >= sc->eblock && end < sc->eend) - end = sc->eend; + if (block > part->eblock && block <= part->eend) + block = part->eblock; + if (end >= part->eblock && end < part->eend) + end = part->eend; /* Safe round to the erase sector boundaries. */ erase_sector = mmc_get_erase_sector(dev); start = block + erase_sector - 1; /* Round up. */ start -= start % erase_sector; stop = end; /* Round down. */ - stop -= end % erase_sector; - /* We can't erase area smaller then sector, store it for later. */ + stop -= end % erase_sector; + /* We can't erase an area smaller than a sector, store it for later. */ if (start >= stop) { - sc->eblock = block; - sc->eend = end; + part->eblock = block; + part->eend = end; return (end); } @@ -720,40 +1378,54 @@ mmcsd_delete(struct mmcsd_softc *sc, struct bio *bp) return (block); } /* Store one of remaining parts for the next call. */ - if (bp->bio_pblkno >= sc->eblock || block == start) { - sc->eblock = stop; /* Predict next forward. */ - sc->eend = end; + if (bp->bio_pblkno >= part->eblock || block == start) { + part->eblock = stop; /* Predict next forward. */ + part->eend = end; } else { - sc->eblock = block; /* Predict next backward. */ - sc->eend = start; + part->eblock = block; /* Predict next backward. */ + part->eend = start; } return (end); } static int -mmcsd_dump(void *arg, void *virtual, vm_offset_t physical, - off_t offset, size_t length) +mmcsd_dump(void *arg, void *virtual, vm_offset_t physical, off_t offset, + size_t length) { - struct disk *disk = arg; - struct mmcsd_softc *sc = (struct mmcsd_softc *)disk->d_drv1; - device_t dev = sc->dev; struct bio bp; daddr_t block, end; - device_t mmcbr = device_get_parent(dev); + struct disk *disk; + struct mmcsd_softc *sc; + struct mmcsd_part *part; + device_t dev, mmcbr; + int err; /* length zero is special and really means flush buffers to media */ if (!length) return (0); + disk = arg; + part = disk->d_drv1; + sc = part->sc; + dev = sc->dev; + mmcbr = sc->mmcbr; + g_reset_bio(&bp); bp.bio_disk = disk; bp.bio_pblkno = offset / disk->d_sectorsize; bp.bio_bcount = length; bp.bio_data = virtual; bp.bio_cmd = BIO_WRITE; - end = bp.bio_pblkno + bp.bio_bcount / sc->disk->d_sectorsize; + end = bp.bio_pblkno + bp.bio_bcount / disk->d_sectorsize; MMCBUS_ACQUIRE_BUS(mmcbr, dev); - block = mmcsd_rw(sc, &bp); + err = mmcsd_switch_part(mmcbr, dev, sc->rca, part->type); + if (err != MMC_ERR_NONE) { + if (ppsratecheck(&sc->log_time, &sc->log_count, LOG_PPS)) + device_printf(dev, "Partition switch error\n"); + MMCBUS_RELEASE_BUS(mmcbr, dev); + return (EIO); + } + block = mmcsd_rw(part, &bp); MMCBUS_RELEASE_BUS(mmcbr, dev); return ((end < block) ? EIO : 0); } @@ -761,24 +1433,30 @@ mmcsd_dump(void *arg, void *virtual, vm_offset_t physical, static void mmcsd_task(void *arg) { - struct mmcsd_softc *sc = (struct mmcsd_softc*)arg; - struct bio *bp; - int sz; daddr_t block, end; - device_t dev = sc->dev; - device_t mmcbr = device_get_parent(sc->dev); + struct mmcsd_part *part; + struct mmcsd_softc *sc; + struct bio *bp; + device_t dev, mmcbr; + int err, sz; + + part = arg; + sc = part->sc; + dev = sc->dev; + mmcbr = sc->mmcbr; while (1) { - MMCSD_LOCK(sc); + MMCSD_PART_LOCK(part); do { - if (sc->running == 0) + if (part->running == 0) goto out; - bp = bioq_takefirst(&sc->bio_queue); + bp = bioq_takefirst(&part->bio_queue); if (bp == NULL) - msleep(sc, &sc->sc_mtx, PRIBIO, "jobqueue", 0); + msleep(part, &part->part_mtx, PRIBIO, + "jobqueue", 0); } while (bp == NULL); - MMCSD_UNLOCK(sc); - if (bp->bio_cmd != BIO_READ && mmc_get_read_only(dev)) { + MMCSD_PART_UNLOCK(part); + if (bp->bio_cmd != BIO_READ && part->ro) { bp->bio_error = EROFS; bp->bio_resid = bp->bio_bcount; bp->bio_flags |= BIO_ERROR; @@ -786,30 +1464,40 @@ mmcsd_task(void *arg) continue; } MMCBUS_ACQUIRE_BUS(mmcbr, dev); - sz = sc->disk->d_sectorsize; + sz = part->disk->d_sectorsize; block = bp->bio_pblkno; end = bp->bio_pblkno + (bp->bio_bcount / sz); + err = mmcsd_switch_part(mmcbr, dev, sc->rca, part->type); + if (err != MMC_ERR_NONE) { + if (ppsratecheck(&sc->log_time, &sc->log_count, + LOG_PPS)) + device_printf(dev, "Partition switch error\n"); + goto release; + } if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) { /* Access to the remaining erase block obsoletes it. */ - if (block < sc->eend && end > sc->eblock) - sc->eblock = sc->eend = 0; - block = mmcsd_rw(sc, bp); + if (block < part->eend && end > part->eblock) + part->eblock = part->eend = 0; + block = mmcsd_rw(part, bp); } else if (bp->bio_cmd == BIO_DELETE) { - block = mmcsd_delete(sc, bp); + block = mmcsd_delete(part, bp); } +release: MMCBUS_RELEASE_BUS(mmcbr, dev); if (block < end) { bp->bio_error = EIO; bp->bio_resid = (end - block) * sz; bp->bio_flags |= BIO_ERROR; + } else { + bp->bio_resid = 0; } biodone(bp); } out: /* tell parent we're done */ - sc->running = -1; - MMCSD_UNLOCK(sc); - wakeup(sc); + part->running = -1; + MMCSD_PART_UNLOCK(part); + wakeup(part); kproc_exit(0); } @@ -842,4 +1530,24 @@ static driver_t mmcsd_driver = { }; static devclass_t mmcsd_devclass; -DRIVER_MODULE(mmcsd, mmc, mmcsd_driver, mmcsd_devclass, NULL, NULL); +static int +mmcsd_handler(module_t mod __unused, int what, void *arg __unused) +{ + +#ifndef __rtems__ + switch (what) { + case MOD_LOAD: + flash_register_slicer(mmcsd_slicer, FLASH_SLICES_TYPE_MMC, + TRUE); + return (0); + case MOD_UNLOAD: + flash_register_slicer(NULL, FLASH_SLICES_TYPE_MMC, TRUE); + return (0); + } +#endif /* __rtems__ */ + return (0); +} + +DRIVER_MODULE(mmcsd, mmc, mmcsd_driver, mmcsd_devclass, mmcsd_handler, NULL); +MODULE_DEPEND(mmcsd, g_flashmap, 0, 0, 0); +MMC_DEPEND(mmcsd); diff --git a/freebsd/sys/dev/mmc/mmcvar.h b/freebsd/sys/dev/mmc/mmcvar.h index c7a4af99..9f62b112 100644 --- a/freebsd/sys/dev/mmc/mmcvar.h +++ b/freebsd/sys/dev/mmc/mmcvar.h @@ -49,15 +49,14 @@ * or the SD Card Association to disclose or distribute any technical * information, know-how or other confidential information to any third party. * - * "$FreeBSD$" + * $FreeBSD$ */ #ifndef DEV_MMC_MMCVAR_H #define DEV_MMC_MMCVAR_H -#include <dev/mmc/bridge.h> - enum mmc_device_ivars { + MMC_IVAR_SPEC_VERS, MMC_IVAR_DSR_IMP, MMC_IVAR_MEDIA_SIZE, MMC_IVAR_RCA, @@ -79,6 +78,7 @@ enum mmc_device_ivars { #define MMC_ACCESSOR(var, ivar, type) \ __BUS_ACCESSOR(mmc, var, MMC, ivar, type) +MMC_ACCESSOR(spec_vers, SPEC_VERS, uint8_t) MMC_ACCESSOR(dsr_imp, DSR_IMP, int) MMC_ACCESSOR(media_size, MEDIA_SIZE, long) MMC_ACCESSOR(rca, RCA, int) diff --git a/freebsd/sys/dev/nvme/nvme.h b/freebsd/sys/dev/nvme/nvme.h index 9c1dab17..1bad4392 100644 --- a/freebsd/sys/dev/nvme/nvme.h +++ b/freebsd/sys/dev/nvme/nvme.h @@ -955,7 +955,8 @@ void nvme_ns_rw_cmd(struct nvme_command *cmd, uint32_t rwcmd, uint16_t nsid, { cmd->opc = rwcmd; cmd->nsid = nsid; - *(uint64_t *)&cmd->cdw10 = lba; + cmd->cdw10 = lba & 0xffffffffu; + cmd->cdw11 = lba >> 32; cmd->cdw12 = count-1; cmd->cdw13 = 0; cmd->cdw14 = 0; diff --git a/freebsd/sys/dev/ofw/ofw_bus_subr.c b/freebsd/sys/dev/ofw/ofw_bus_subr.c index 79852afc..a067b949 100644 --- a/freebsd/sys/dev/ofw/ofw_bus_subr.c +++ b/freebsd/sys/dev/ofw/ofw_bus_subr.c @@ -48,6 +48,7 @@ __FBSDID("$FreeBSD$"); #include <rtems/bsd/local/ofw_bus_if.h> #define OFW_COMPAT_LEN 255 +#define OFW_STATUS_LEN 16 int ofw_bus_gen_setup_devinfo(struct ofw_bus_devinfo *obd, phandle_t node) @@ -181,6 +182,24 @@ ofw_bus_status_okay(device_t dev) return (0); } +int +ofw_bus_node_status_okay(phandle_t node) +{ + char status[OFW_STATUS_LEN]; + int len; + + len = OF_getproplen(node, "status"); + if (len <= 0) + return (1); + + OF_getprop(node, "status", status, OFW_STATUS_LEN); + if ((len == 5 && (bcmp(status, "okay", len) == 0)) || + (len == 3 && (bcmp(status, "ok", len)))) + return (1); + + return (0); +} + static int ofw_bus_node_is_compatible_int(const char *compat, int len, const char *onecompat) @@ -946,7 +965,7 @@ ofw_bus_string_list_to_array(phandle_t node, const char *list_name, i += len; tptr += len; } - array[cnt] = 0; + array[cnt] = NULL; *out_array = array; return (cnt); diff --git a/freebsd/sys/dev/ofw/ofw_bus_subr.h b/freebsd/sys/dev/ofw/ofw_bus_subr.h index 30f299a6..4afd84e3 100644 --- a/freebsd/sys/dev/ofw/ofw_bus_subr.h +++ b/freebsd/sys/dev/ofw/ofw_bus_subr.h @@ -100,6 +100,7 @@ int ofw_bus_intr_by_rid(device_t, phandle_t, int, phandle_t *, int *, /* Helper to get device status property */ const char *ofw_bus_get_status(device_t dev); int ofw_bus_status_okay(device_t dev); +int ofw_bus_node_status_okay(phandle_t node); /* Helper to get node's interrupt parent */ phandle_t ofw_bus_find_iparent(phandle_t); diff --git a/freebsd/sys/dev/ofw/ofw_fdt.c b/freebsd/sys/dev/ofw/ofw_fdt.c index ae3da8e4..20e07e90 100644 --- a/freebsd/sys/dev/ofw/ofw_fdt.c +++ b/freebsd/sys/dev/ofw/ofw_fdt.c @@ -432,6 +432,7 @@ ofw_fdt_package_to_path(ofw_t ofw, phandle_t package, char *buf, size_t len) return (-1); } +#ifndef __rtems__ #if defined(FDT_MARVELL) || defined(__powerpc__) static int ofw_fdt_fixup(ofw_t ofw) @@ -467,11 +468,13 @@ ofw_fdt_fixup(ofw_t ofw) return (0); } #endif +#endif /* __rtems__ */ static int ofw_fdt_interpret(ofw_t ofw, const char *cmd, int nret, cell_t *retvals) { #if defined(FDT_MARVELL) || defined(__powerpc__) +#ifndef __rtems__ int rv; /* @@ -490,6 +493,9 @@ ofw_fdt_interpret(ofw_t ofw, const char *cmd, int nret, cell_t *retvals) retvals[0] = rv; return (rv); +#else /* __rtems__ */ + return (0); +#endif /* __rtems__ */ #else return (0); #endif diff --git a/freebsd/sys/dev/pci/pci.c b/freebsd/sys/dev/pci/pci.c index 2eba4ca2..3209f893 100644 --- a/freebsd/sys/dev/pci/pci.c +++ b/freebsd/sys/dev/pci/pci.c @@ -283,13 +283,14 @@ static const struct pci_quirk pci_quirks[] = { { 0x43851002, PCI_QUIRK_UNMAP_REG, 0x14, 0 }, /* - * Atheros AR8161/AR8162/E2200/E2400 Ethernet controllers have a - * bug that MSI interrupt does not assert if PCIM_CMD_INTxDIS bit + * Atheros AR8161/AR8162/E2200/E2400/E2500 Ethernet controllers have + * a bug that MSI interrupt does not assert if PCIM_CMD_INTxDIS bit * of the command register is set. */ { 0x10911969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, { 0xE0911969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, { 0xE0A11969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, + { 0xE0B11969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, { 0x10901969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* @@ -3099,7 +3100,7 @@ pci_add_map(device_t bus, device_t dev, int reg, struct resource_list *rl, * If base is 0, then we have problems if this architecture does * not allow that. It is best to ignore such entries for the * moment. These will be allocated later if the driver specifically - * requests them. However, some removable busses look better when + * requests them. However, some removable buses look better when * all resources are allocated, so allow '0' to be overriden. * * Similarly treat maps whose values is the same as the test value @@ -4188,7 +4189,7 @@ pci_attach(device_t dev) /* * Since there can be multiple independently numbered PCI - * busses on systems with multiple PCI domains, we can't use + * buses on systems with multiple PCI domains, we can't use * the unit number to decide which bus we are probing. We ask * the parent pcib what our domain and bus numbers are. */ diff --git a/freebsd/sys/dev/pci/pci_pci.c b/freebsd/sys/dev/pci/pci_pci.c index 7d763dd9..d468ca2e 100644 --- a/freebsd/sys/dev/pci/pci_pci.c +++ b/freebsd/sys/dev/pci/pci_pci.c @@ -78,6 +78,8 @@ static void pcib_pcie_ab_timeout(void *arg); static void pcib_pcie_cc_timeout(void *arg); static void pcib_pcie_dll_timeout(void *arg); #endif +static int pcib_request_feature(device_t pcib, device_t dev, + enum pci_feature feature); static device_method_t pcib_methods[] = { /* Device interface */ @@ -121,6 +123,7 @@ static device_method_t pcib_methods[] = { DEVMETHOD(pcib_try_enable_ari, pcib_try_enable_ari), DEVMETHOD(pcib_ari_enabled, pcib_ari_enabled), DEVMETHOD(pcib_decode_rid, pcib_ari_decode_rid), + DEVMETHOD(pcib_request_feature, pcib_request_feature), DEVMETHOD_END }; @@ -920,6 +923,7 @@ static void pcib_probe_hotplug(struct pcib_softc *sc) { device_t dev; + uint32_t link_cap; uint16_t link_sta, slot_sta; if (!pci_enable_pcie_hp) @@ -932,11 +936,13 @@ pcib_probe_hotplug(struct pcib_softc *sc) if (!(pcie_read_config(dev, PCIER_FLAGS, 2) & PCIEM_FLAGS_SLOT)) return; - sc->pcie_link_cap = pcie_read_config(dev, PCIER_LINK_CAP, 4); sc->pcie_slot_cap = pcie_read_config(dev, PCIER_SLOT_CAP, 4); if ((sc->pcie_slot_cap & PCIEM_SLOT_CAP_HPC) == 0) return; + link_cap = pcie_read_config(dev, PCIER_LINK_CAP, 4); + if ((link_cap & PCIEM_LINK_CAP_DL_ACTIVE) == 0) + return; /* * Some devices report that they have an MRL when they actually @@ -947,8 +953,7 @@ pcib_probe_hotplug(struct pcib_softc *sc) * If there is an open MRL but the Data Link Layer is active, * the MRL is not real. */ - if ((sc->pcie_slot_cap & PCIEM_SLOT_CAP_MRLSP) != 0 && - (sc->pcie_link_cap & PCIEM_LINK_CAP_DL_ACTIVE) != 0) { + if ((sc->pcie_slot_cap & PCIEM_SLOT_CAP_MRLSP) != 0) { link_sta = pcie_read_config(dev, PCIER_LINK_STA, 2); slot_sta = pcie_read_config(dev, PCIER_SLOT_STA, 2); if ((slot_sta & PCIEM_SLOT_STA_MRLSS) != 0 && @@ -957,6 +962,17 @@ pcib_probe_hotplug(struct pcib_softc *sc) } } + /* + * Now that we're sure we want to do hot plug, ask the + * firmware, if any, if that's OK. + */ + if (pcib_request_feature(device_get_parent(device_get_parent(dev)), dev, + PCI_FEATURE_HP) != 0) { + if (bootverbose) + device_printf(dev, "Unable to activate hot plug feature.\n"); + return; + } + sc->flags |= PCIB_HOTPLUG; } @@ -1061,10 +1077,8 @@ pcib_hotplug_present(struct pcib_softc *sc) return (0); /* Require the Data Link Layer to be active. */ - if (sc->pcie_link_cap & PCIEM_LINK_CAP_DL_ACTIVE) { - if (!(sc->pcie_link_sta & PCIEM_LINK_STA_DL_ACTIVE)) - return (0); - } + if (!(sc->pcie_link_sta & PCIEM_LINK_STA_DL_ACTIVE)) + return (0); return (-1); } @@ -1121,20 +1135,18 @@ pcib_pcie_hotplug_update(struct pcib_softc *sc, uint16_t val, uint16_t mask, * changed on this interrupt. Stop any scheduled timer if * the Data Link Layer is active. */ - if (sc->pcie_link_cap & PCIEM_LINK_CAP_DL_ACTIVE) { - if (card_inserted && - !(sc->pcie_link_sta & PCIEM_LINK_STA_DL_ACTIVE) && - sc->pcie_slot_sta & - (PCIEM_SLOT_STA_MRLSC | PCIEM_SLOT_STA_PDC)) { - if (cold) - device_printf(sc->dev, - "Data Link Layer inactive\n"); - else - callout_reset(&sc->pcie_dll_timer, hz, - pcib_pcie_dll_timeout, sc); - } else if (sc->pcie_link_sta & PCIEM_LINK_STA_DL_ACTIVE) - callout_stop(&sc->pcie_dll_timer); - } + if (card_inserted && + !(sc->pcie_link_sta & PCIEM_LINK_STA_DL_ACTIVE) && + sc->pcie_slot_sta & + (PCIEM_SLOT_STA_MRLSC | PCIEM_SLOT_STA_PDC)) { + if (cold) + device_printf(sc->dev, + "Data Link Layer inactive\n"); + else + callout_reset(&sc->pcie_dll_timer, hz, + pcib_pcie_dll_timeout, sc); + } else if (sc->pcie_link_sta & PCIEM_LINK_STA_DL_ACTIVE) + callout_stop(&sc->pcie_dll_timer); pcib_pcie_hotplug_command(sc, val, mask); @@ -1149,7 +1161,7 @@ pcib_pcie_hotplug_update(struct pcib_softc *sc, uint16_t val, uint16_t mask, } static void -pcib_pcie_intr(void *arg) +pcib_pcie_intr_hotplug(void *arg) { struct pcib_softc *sc; device_t dev; @@ -1262,7 +1274,7 @@ pcib_pcie_cc_timeout(void *arg) } else { device_printf(dev, "Missed HotPlug interrupt waiting for Command Completion\n"); - pcib_pcie_intr(sc); + pcib_pcie_intr_hotplug(sc); } } @@ -1285,7 +1297,7 @@ pcib_pcie_dll_timeout(void *arg) } else if (sta != sc->pcie_link_sta) { device_printf(dev, "Missed HotPlug interrupt waiting for DLL Active\n"); - pcib_pcie_intr(sc); + pcib_pcie_intr_hotplug(sc); } } @@ -1331,7 +1343,7 @@ pcib_alloc_pcie_irq(struct pcib_softc *sc) } error = bus_setup_intr(dev, sc->pcie_irq, INTR_TYPE_MISC, - NULL, pcib_pcie_intr, sc, &sc->pcie_ihand); + NULL, pcib_pcie_intr_hotplug, sc, &sc->pcie_ihand); if (error) { device_printf(dev, "Failed to setup PCI-e interrupt handler\n"); bus_release_resource(dev, SYS_RES_IRQ, rid, sc->pcie_irq); @@ -1384,7 +1396,7 @@ pcib_setup_hotplug(struct pcib_softc *sc) mask = PCIEM_SLOT_CTL_DLLSCE | PCIEM_SLOT_CTL_HPIE | PCIEM_SLOT_CTL_CCIE | PCIEM_SLOT_CTL_PDCE | PCIEM_SLOT_CTL_MRLSCE | PCIEM_SLOT_CTL_PFDE | PCIEM_SLOT_CTL_ABPE; - val = PCIEM_SLOT_CTL_PDCE | PCIEM_SLOT_CTL_HPIE; + val = PCIEM_SLOT_CTL_DLLSCE | PCIEM_SLOT_CTL_HPIE | PCIEM_SLOT_CTL_PDCE; if (sc->pcie_slot_cap & PCIEM_SLOT_CAP_APB) val |= PCIEM_SLOT_CTL_ABPE; if (sc->pcie_slot_cap & PCIEM_SLOT_CAP_PCP) @@ -1393,8 +1405,6 @@ pcib_setup_hotplug(struct pcib_softc *sc) val |= PCIEM_SLOT_CTL_MRLSCE; if (!(sc->pcie_slot_cap & PCIEM_SLOT_CAP_NCCS)) val |= PCIEM_SLOT_CTL_CCIE; - if (sc->pcie_link_cap & PCIEM_LINK_CAP_DL_ACTIVE) - val |= PCIEM_SLOT_CTL_DLLSCE; /* Turn the attention indicator off. */ if (sc->pcie_slot_cap & PCIEM_SLOT_CAP_AIP) { @@ -2835,3 +2845,43 @@ pcib_try_enable_ari(device_t pcib, device_t dev) return (0); } + +int +pcib_request_feature_allow(device_t pcib, device_t dev, + enum pci_feature feature) +{ + /* + * No host firmware we have to negotiate with, so we allow + * every valid feature requested. + */ + switch (feature) { + case PCI_FEATURE_AER: + case PCI_FEATURE_HP: + break; + default: + return (EINVAL); + } + + return (0); +} + +/* + * Pass the request to use this PCI feature up the tree. Either there's a + * firmware like ACPI that's using this feature that will approve (or deny) the + * request to take it over, or the platform has no such firmware, in which case + * the request will be approved. If the request is approved, the OS is expected + * to make use of the feature or render it harmless. + */ +static int +pcib_request_feature(device_t pcib, device_t dev, enum pci_feature feature) +{ + device_t bus; + + /* + * Our parent is necessarily a pci bus. Its parent will either be + * another pci bridge (which passes it up) or a host bridge that can + * approve or reject the request. + */ + bus = device_get_parent(pcib); + return (PCIB_REQUEST_FEATURE(device_get_parent(bus), dev, feature)); +} diff --git a/freebsd/sys/dev/pci/pci_private.h b/freebsd/sys/dev/pci/pci_private.h index b0f14818..6c5a1677 100644 --- a/freebsd/sys/dev/pci/pci_private.h +++ b/freebsd/sys/dev/pci/pci_private.h @@ -34,7 +34,7 @@ /* * Export definitions of the pci bus so that we can more easily share - * it with "subclass" busses. + * it with "subclass" buses. */ DECLARE_CLASS(pci_driver); diff --git a/freebsd/sys/dev/pci/pcib_private.h b/freebsd/sys/dev/pci/pcib_private.h index 65aec8d4..1004e133 100644 --- a/freebsd/sys/dev/pci/pcib_private.h +++ b/freebsd/sys/dev/pci/pcib_private.h @@ -132,7 +132,6 @@ struct pcib_softc uint16_t bridgectl; /* bridge control register */ uint16_t pcie_link_sta; uint16_t pcie_slot_sta; - uint32_t pcie_link_cap; uint32_t pcie_slot_cap; struct resource *pcie_irq; void *pcie_ihand; @@ -194,5 +193,6 @@ int pcib_get_id(device_t pcib, device_t dev, enum pci_id_type type, uintptr_t *id); void pcib_decode_rid(device_t pcib, uint16_t rid, int *bus, int *slot, int *func); +int pcib_request_feature_allow(device_t pcib, device_t dev, enum pci_feature feature); #endif diff --git a/freebsd/sys/dev/pci/pcireg.h b/freebsd/sys/dev/pci/pcireg.h index 291bb2ea..b434b2e6 100644 --- a/freebsd/sys/dev/pci/pcireg.h +++ b/freebsd/sys/dev/pci/pcireg.h @@ -478,6 +478,11 @@ #define PCIS_DASP_MGMT_CARD 0x20 #define PCIS_DASP_OTHER 0x80 +#define PCIC_ACCEL 0x12 +#define PCIS_ACCEL_PROCESSING 0x00 + +#define PCIC_INSTRUMENT 0x13 + #define PCIC_OTHER 0xff /* Bridge Control Values. */ @@ -1040,3 +1045,19 @@ #define PCIR_SRIOV_BARS 0x24 #define PCIR_SRIOV_BAR(x) (PCIR_SRIOV_BARS + (x) * 4) +/* + * PCI Express Firmware Interface definitions + */ +#define PCI_OSC_STATUS 0 +#define PCI_OSC_SUPPORT 1 +#define PCIM_OSC_SUPPORT_EXT_PCI_CONF 0x01 /* Extended PCI Config Space */ +#define PCIM_OSC_SUPPORT_ASPM 0x02 /* Active State Power Management */ +#define PCIM_OSC_SUPPORT_CPMC 0x04 /* Clock Power Management Cap */ +#define PCIM_OSC_SUPPORT_SEG_GROUP 0x08 /* PCI Segment Groups supported */ +#define PCIM_OSC_SUPPORT_MSI 0x10 /* MSI signalling supported */ +#define PCI_OSC_CTL 2 +#define PCIM_OSC_CTL_PCIE_HP 0x01 /* PCIe Native Hot Plug */ +#define PCIM_OSC_CTL_SHPC_HP 0x02 /* SHPC Native Hot Plug */ +#define PCIM_OSC_CTL_PCIE_PME 0x04 /* PCIe Native Power Mgt Events */ +#define PCIM_OSC_CTL_PCIE_AER 0x08 /* PCIe Advanced Error Reporting */ +#define PCIM_OSC_CTL_PCIE_CAP_STRUCT 0x10 /* Various Capability Structures */ diff --git a/freebsd/sys/dev/rtwn/if_rtwn.c b/freebsd/sys/dev/rtwn/if_rtwn.c index a553814f..050d9960 100644 --- a/freebsd/sys/dev/rtwn/if_rtwn.c +++ b/freebsd/sys/dev/rtwn/if_rtwn.c @@ -123,9 +123,6 @@ static int rtwn_run(struct rtwn_softc *, static void rtwn_watchdog(void *); #endif static void rtwn_parent(struct ieee80211com *); -static int rtwn_llt_write(struct rtwn_softc *, uint32_t, - uint32_t); -static int rtwn_llt_init(struct rtwn_softc *); static int rtwn_dma_init(struct rtwn_softc *); static int rtwn_mac_init(struct rtwn_softc *); static void rtwn_mrr_init(struct rtwn_softc *); @@ -697,6 +694,7 @@ rtwn_ioctl_reset(struct ieee80211vap *vap, u_long cmd) case IEEE80211_IOC_RTSTHRESHOLD: case IEEE80211_IOC_PROTMODE: case IEEE80211_IOC_HTPROTMODE: + case IEEE80211_IOC_LDPC: error = 0; break; default: @@ -1384,54 +1382,6 @@ rtwn_parent(struct ieee80211com *ic) rtwn_stop(sc); } - -static int -rtwn_llt_write(struct rtwn_softc *sc, uint32_t addr, uint32_t data) -{ - int ntries, error; - - error = rtwn_write_4(sc, R92C_LLT_INIT, - SM(R92C_LLT_INIT_OP, R92C_LLT_INIT_OP_WRITE) | - SM(R92C_LLT_INIT_ADDR, addr) | - SM(R92C_LLT_INIT_DATA, data)); - if (error != 0) - return (error); - /* Wait for write operation to complete. */ - for (ntries = 0; ntries < 20; ntries++) { - if (MS(rtwn_read_4(sc, R92C_LLT_INIT), R92C_LLT_INIT_OP) == - R92C_LLT_INIT_OP_NO_ACTIVE) - return (0); - rtwn_delay(sc, 10); - } - return (ETIMEDOUT); -} - -static int -rtwn_llt_init(struct rtwn_softc *sc) -{ - int i, error; - - /* Reserve pages [0; page_count]. */ - for (i = 0; i < sc->page_count; i++) { - if ((error = rtwn_llt_write(sc, i, i + 1)) != 0) - return (error); - } - /* NB: 0xff indicates end-of-list. */ - if ((error = rtwn_llt_write(sc, i, 0xff)) != 0) - return (error); - /* - * Use pages [page_count + 1; pktbuf_count - 1] - * as ring buffer. - */ - for (++i; i < sc->pktbuf_count - 1; i++) { - if ((error = rtwn_llt_write(sc, i, i + 1)) != 0) - return (error); - } - /* Make the last page point to the beginning of the ring buffer. */ - error = rtwn_llt_write(sc, i, sc->page_count + 1); - return (error); -} - static int rtwn_dma_init(struct rtwn_softc *sc) { @@ -1770,13 +1720,13 @@ rtwn_node_alloc(struct ieee80211vap *vap, } static void -rtwn_newassoc(struct ieee80211_node *ni, int isnew) +rtwn_newassoc(struct ieee80211_node *ni, int isnew __unused) { struct rtwn_softc *sc = ni->ni_ic->ic_softc; struct rtwn_node *un = RTWN_NODE(ni); int id; - if (!isnew) + if (un->id != RTWN_MACID_UNDEFINED) return; RTWN_NT_LOCK(sc); @@ -2001,6 +1951,7 @@ rtwn_stop(struct rtwn_softc *sc) sc->fwver = 0; sc->thcal_temp = 0; sc->cur_bcnq_id = RTWN_VAP_ID_INVALID; + bzero(&sc->last_physt, sizeof(sc->last_physt)); #ifdef D4054 ieee80211_tx_watchdog_stop(&sc->sc_ic); diff --git a/freebsd/sys/dev/rtwn/if_rtwn_rx.c b/freebsd/sys/dev/rtwn/if_rtwn_rx.c index 8d103dc7..31ab7e69 100644 --- a/freebsd/sys/dev/rtwn/if_rtwn_rx.c +++ b/freebsd/sys/dev/rtwn/if_rtwn_rx.c @@ -119,18 +119,19 @@ rtwn_set_basicrates(struct rtwn_softc *sc, uint32_t rates) } static void -rtwn_update_avgrssi(struct rtwn_softc *sc, struct rtwn_node *un, int rate) +rtwn_update_avgrssi(struct rtwn_softc *sc, struct rtwn_node *un, int8_t rssi, + int is_cck) { int pwdb; /* Convert antenna signal to percentage. */ - if (un->last_rssi <= -100 || un->last_rssi >= 20) + if (rssi <= -100 || rssi >= 20) pwdb = 0; - else if (un->last_rssi >= 0) + else if (rssi >= 0) pwdb = 100; else - pwdb = 100 + un->last_rssi; - if (RTWN_RATE_IS_CCK(rate)) { + pwdb = 100 + rssi; + if (is_cck) { /* CCK gain is smaller than OFDM/MCS gain. */ pwdb += 6; if (pwdb > 100) @@ -157,11 +158,11 @@ rtwn_update_avgrssi(struct rtwn_softc *sc, struct rtwn_node *un, int rate) } static int8_t -rtwn_get_rssi(struct rtwn_softc *sc, int rate, void *physt) +rtwn_get_rssi(struct rtwn_softc *sc, void *physt, int is_cck) { int8_t rssi; - if (RTWN_RATE_IS_CCK(rate)) + if (is_cck) rssi = rtwn_get_rssi_cck(sc, physt); else /* OFDM/HT. */ rssi = rtwn_get_rssi_ofdm(sc, physt); @@ -190,81 +191,133 @@ rtwn_get_tsf(struct rtwn_softc *sc, uint64_t *buf, int id) *buf += rtwn_get_tsf_low(sc, id); } +static uint64_t +rtwn_extend_rx_tsf(struct rtwn_softc *sc, const struct r92c_rx_stat *stat) +{ + uint64_t tsft; + uint32_t rxdw3, tsfl, tsfl_curr; + int id; + + rxdw3 = le32toh(stat->rxdw3); + tsfl = le32toh(stat->tsf_low); + id = MS(rxdw3, R92C_RXDW3_BSSID_FIT); + + switch (id) { + case 1: + case 2: + id >>= 1; + tsfl_curr = rtwn_get_tsf_low(sc, id); + break; + default: + { + uint32_t tsfl0, tsfl1; + + tsfl0 = rtwn_get_tsf_low(sc, 0); + tsfl1 = rtwn_get_tsf_low(sc, 1); + + if (abs(tsfl0 - tsfl) < abs(tsfl1 - tsfl)) { + id = 0; + tsfl_curr = tsfl0; + } else { + id = 1; + tsfl_curr = tsfl1; + } + break; + } + } + + tsft = rtwn_get_tsf_high(sc, id); + if (tsfl > tsfl_curr && tsfl > 0xffff0000) + tsft--; + tsft <<= 32; + tsft += tsfl; + + return (tsft); +} + struct ieee80211_node * -rtwn_rx_common(struct rtwn_softc *sc, struct mbuf *m, void *desc, - int8_t *rssi) +rtwn_rx_common(struct rtwn_softc *sc, struct mbuf *m, void *desc) { struct ieee80211com *ic = &sc->sc_ic; struct ieee80211_node *ni; struct ieee80211_frame_min *wh; + struct ieee80211_rx_stats rxs; struct rtwn_node *un; struct r92c_rx_stat *stat; - uint32_t rxdw0, rxdw3; - int cipher, infosz, pktlen, rate, shift; + void *physt; + uint32_t rxdw0; + int8_t rssi; + int cipher, infosz, is_cck, pktlen, shift; stat = desc; rxdw0 = le32toh(stat->rxdw0); - rxdw3 = le32toh(stat->rxdw3); cipher = MS(rxdw0, R92C_RXDW0_CIPHER); infosz = MS(rxdw0, R92C_RXDW0_INFOSZ) * 8; pktlen = MS(rxdw0, R92C_RXDW0_PKTLEN); shift = MS(rxdw0, R92C_RXDW0_SHIFT); - rate = MS(rxdw3, R92C_RXDW3_RATE); wh = (struct ieee80211_frame_min *)(mtodo(m, shift + infosz)); if ((wh->i_fc[1] & IEEE80211_FC1_PROTECTED) && cipher != R92C_CAM_ALGO_NONE) m->m_flags |= M_WEP; - if (pktlen >= sizeof(*wh)) + if (pktlen >= sizeof(*wh)) { ni = ieee80211_find_rxnode(ic, wh); - else + if (ni != NULL && (ni->ni_flags & IEEE80211_NODE_HT)) + m->m_flags |= M_AMPDU; + } else ni = NULL; un = RTWN_NODE(ni); - /* Get RSSI from PHY status descriptor if present. */ - if (infosz != 0 && (rxdw0 & R92C_RXDW0_PHYST)) { - *rssi = rtwn_get_rssi(sc, rate, mtod(m, void *)); - RTWN_DPRINTF(sc, RTWN_DEBUG_RSSI, "%s: rssi %d, ridx %d\n", - __func__, *rssi, rate); + if (infosz != 0 && (rxdw0 & R92C_RXDW0_PHYST)) + physt = (void *)mtodo(m, shift); + else + physt = (un != NULL) ? &un->last_physt : &sc->last_physt; + + bzero(&rxs, sizeof(rxs)); + rtwn_get_rx_stats(sc, &rxs, desc, physt); + if (rxs.c_pktflags & IEEE80211_RX_F_AMPDU) { + /* Next MPDU will come without PHY info. */ + memcpy(&sc->last_physt, physt, sizeof(sc->last_physt)); + if (un != NULL) + memcpy(&un->last_physt, physt, sizeof(sc->last_physt)); + } - sc->last_rssi = *rssi; - if (un != NULL) { - un->last_rssi = *rssi; + /* Add some common bits. */ + /* NB: should not happen. */ + if (rxdw0 & R92C_RXDW0_CRCERR) + rxs.c_pktflags |= IEEE80211_RX_F_FAIL_FCSCRC; + + rxs.r_flags |= IEEE80211_R_TSF_START; /* XXX undocumented */ + rxs.r_flags |= IEEE80211_R_TSF64; + rxs.c_rx_tsf = rtwn_extend_rx_tsf(sc, stat); + + /* Get RSSI from PHY status descriptor. */ + is_cck = (rxs.c_pktflags & IEEE80211_RX_F_CCK) != 0; + rssi = rtwn_get_rssi(sc, physt, is_cck); + + /* XXX TODO: we really need a rate-to-string method */ + RTWN_DPRINTF(sc, RTWN_DEBUG_RSSI, "%s: rssi %d, rate %d\n", + __func__, rssi, rxs.c_rate); + if (un != NULL && infosz != 0 && (rxdw0 & R92C_RXDW0_PHYST)) { + /* Update our average RSSI. */ + rtwn_update_avgrssi(sc, un, rssi, is_cck); + } - /* Update our average RSSI. */ - rtwn_update_avgrssi(sc, un, rate); - } - } else - *rssi = (un != NULL) ? un->last_rssi : sc->last_rssi; + rxs.r_flags |= IEEE80211_R_NF | IEEE80211_R_RSSI; + rxs.c_nf = RTWN_NOISE_FLOOR; + rxs.c_rssi = rssi - rxs.c_nf; + (void) ieee80211_add_rx_params(m, &rxs); if (ieee80211_radiotap_active(ic)) { struct rtwn_rx_radiotap_header *tap = &sc->sc_rxtap; - int id = RTWN_VAP_ID_INVALID; - - if (ni != NULL) - id = RTWN_VAP(ni->ni_vap)->id; - if (id == RTWN_VAP_ID_INVALID) - id = 0; tap->wr_flags = rtwn_rx_radiotap_flags(sc, desc); - tap->wr_tsft = rtwn_get_tsf_high(sc, id); - if (le32toh(stat->tsf_low) > rtwn_get_tsf_low(sc, id)) - tap->wr_tsft--; - tap->wr_tsft = (uint64_t)htole32(tap->wr_tsft) << 32; - tap->wr_tsft += stat->tsf_low; - - /* XXX 20/40? */ - - /* Map HW rate index to 802.11 rate. */ - if (rate < RTWN_RIDX_MCS(0)) - tap->wr_rate = ridx2rate[rate]; - else /* MCS0~15. */ - tap->wr_rate = IEEE80211_RATE_MCS | (rate - 12); - - tap->wr_dbm_antsignal = *rssi; - tap->wr_dbm_antnoise = RTWN_NOISE_FLOOR; + tap->wr_tsft = htole64(rxs.c_rx_tsf); + tap->wr_rate = rxs.c_rate; + tap->wr_dbm_antsignal = rssi; + tap->wr_dbm_antnoise = rxs.c_nf; } /* Drop PHY descriptor. */ diff --git a/freebsd/sys/dev/rtwn/if_rtwn_rx.h b/freebsd/sys/dev/rtwn/if_rtwn_rx.h index dfdcc4bf..49897eb9 100644 --- a/freebsd/sys/dev/rtwn/if_rtwn_rx.h +++ b/freebsd/sys/dev/rtwn/if_rtwn_rx.h @@ -26,7 +26,7 @@ void rtwn_get_rates(struct rtwn_softc *, const struct ieee80211_rateset *, const struct ieee80211_htrateset *, uint32_t *, int *, int); void rtwn_set_basicrates(struct rtwn_softc *, uint32_t); struct ieee80211_node * rtwn_rx_common(struct rtwn_softc *, struct mbuf *, - void *, int8_t *); + void *); void rtwn_adhoc_recv_mgmt(struct ieee80211_node *, struct mbuf *, int, const struct ieee80211_rx_stats *, int, int); void rtwn_set_multi(struct rtwn_softc *); diff --git a/freebsd/sys/dev/rtwn/if_rtwn_tx.c b/freebsd/sys/dev/rtwn/if_rtwn_tx.c index 1ea9a766..c48e2e0e 100644 --- a/freebsd/sys/dev/rtwn/if_rtwn_tx.c +++ b/freebsd/sys/dev/rtwn/if_rtwn_tx.c @@ -114,17 +114,16 @@ static int rtwn_tx_data(struct rtwn_softc *sc, struct ieee80211_node *ni, struct mbuf *m) { - const struct ieee80211_txparam *tp; + const struct ieee80211_txparam *tp = ni->ni_txparms; struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = ni->ni_vap; struct ieee80211_key *k = NULL; - struct ieee80211_channel *chan; struct ieee80211_frame *wh; struct rtwn_tx_desc_common *txd; struct rtwn_tx_buf buf; uint8_t rate, ridx, type; u_int cipher; - int ismcast, maxretry; + int ismcast; RTWN_ASSERT_LOCKED(sc); @@ -132,20 +131,15 @@ rtwn_tx_data(struct rtwn_softc *sc, struct ieee80211_node *ni, type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK; ismcast = IEEE80211_IS_MULTICAST(wh->i_addr1); - chan = (ni->ni_chan != IEEE80211_CHAN_ANYC) ? - ni->ni_chan : ic->ic_curchan; - tp = &vap->iv_txparms[ieee80211_chan2mode(chan)]; - maxretry = tp->maxretry; - /* Choose a TX rate index. */ - if (type == IEEE80211_FC0_TYPE_MGT) + if (type == IEEE80211_FC0_TYPE_MGT || + type == IEEE80211_FC0_TYPE_CTL || + (m->m_flags & M_EAPOL) != 0) rate = tp->mgmtrate; else if (ismcast) rate = tp->mcastrate; else if (tp->ucastrate != IEEE80211_FIXED_RATE_NONE) rate = tp->ucastrate; - else if (m->m_flags & M_EAPOL) - rate = tp->mgmtrate; else { if (sc->sc_ratectl == RTWN_RATECTL_NET80211) { /* XXX pass pktlen */ @@ -183,7 +177,7 @@ rtwn_tx_data(struct rtwn_softc *sc, struct ieee80211_node *ni, memset(txd, 0, sc->txdesc_len); txd->txdw1 = htole32(SM(RTWN_TXDW1_CIPHER, rtwn_get_cipher(cipher))); - rtwn_fill_tx_desc(sc, ni, m, txd, ridx, maxretry); + rtwn_fill_tx_desc(sc, ni, m, txd, ridx, tp->maxretry); if (ieee80211_radiotap_active_vap(vap)) { struct rtwn_tx_radiotap_header *tap = &sc->sc_txtap; diff --git a/freebsd/sys/dev/rtwn/if_rtwnvar.h b/freebsd/sys/dev/rtwn/if_rtwnvar.h index 0c010adb..d8754024 100644 --- a/freebsd/sys/dev/rtwn/if_rtwnvar.h +++ b/freebsd/sys/dev/rtwn/if_rtwnvar.h @@ -76,6 +76,12 @@ struct rtwn_tx_buf { uint8_t txd[RTWN_TX_DESC_SIZE]; } __attribute__((aligned(4))); +#define RTWN_PHY_STATUS_SIZE 32 +struct rtwn_tx_phystat { + uint32_t phydw[RTWN_PHY_STATUS_SIZE / sizeof(uint32_t)]; +}; + + struct rtwn_softc; union sec_param { @@ -95,7 +101,8 @@ struct rtwn_cmdq { struct rtwn_node { struct ieee80211_node ni; /* must be the first */ int id; - int8_t last_rssi; + + struct rtwn_tx_phystat last_physt; int avg_pwdb; }; #define RTWN_NODE(ni) ((struct rtwn_node *)(ni)) @@ -195,7 +202,7 @@ struct rtwn_softc { const char *name; int sc_ant; - int8_t last_rssi; + struct rtwn_tx_phystat last_physt; uint8_t thcal_temp; int cur_bcnq_id; @@ -301,6 +308,7 @@ struct rtwn_softc { void (*sc_fw_reset)(struct rtwn_softc *, int); void (*sc_fw_download_enable)(struct rtwn_softc *, int); #endif + int (*sc_llt_init)(struct rtwn_softc *); int (*sc_set_page_size)(struct rtwn_softc *); void (*sc_lc_calib)(struct rtwn_softc *); void (*sc_iq_calib)(struct rtwn_softc *); @@ -336,6 +344,9 @@ struct rtwn_softc { struct ieee80211vap *, int); void (*sc_set_rssi)(struct rtwn_softc *); #endif + void (*sc_get_rx_stats)(struct rtwn_softc *, + struct ieee80211_rx_stats *, const void *, + const void *); int8_t (*sc_get_rssi_cck)(struct rtwn_softc *, void *); int8_t (*sc_get_rssi_ofdm)(struct rtwn_softc *, void *); int (*sc_classify_intr)(struct rtwn_softc *, void *, int); @@ -462,8 +473,8 @@ void rtwn_suspend(struct rtwn_softc *); /* Aliases. */ #define rtwn_bb_write rtwn_write_4 -#define rtwn_bb_read rtwn_read_4 -#define rtwn_bb_setbits rtwn_setbits_4 +#define rtwn_bb_read rtwn_read_4 +#define rtwn_bb_setbits rtwn_setbits_4 /* Device-specific. */ #define rtwn_rf_read(_sc, _chain, _addr) \ @@ -478,6 +489,8 @@ void rtwn_suspend(struct rtwn_softc *); (((_sc)->sc_parse_rom)((_sc), (_rom))) #define rtwn_set_led(_sc, _led, _on) \ (((_sc)->sc_set_led)((_sc), (_led), (_on))) +#define rtwn_get_rx_stats(_sc, _rxs, _desc, _physt) \ + (((_sc)->sc_get_rx_stats((_sc), (_rxs), (_desc), (_physt)))) #define rtwn_get_rssi_cck(_sc, _physt) \ (((_sc)->sc_get_rssi_cck)((_sc), (_physt))) #define rtwn_get_rssi_ofdm(_sc, _physt) \ @@ -492,6 +505,8 @@ void rtwn_suspend(struct rtwn_softc *); #define rtwn_fw_download_enable(_sc, _enable) \ (((_sc)->sc_fw_download_enable)((_sc), (_enable))) #endif +#define rtwn_llt_init(_sc) \ + (((_sc)->sc_llt_init)((_sc))) #define rtwn_set_page_size(_sc) \ (((_sc)->sc_set_page_size)((_sc))) #define rtwn_lc_calib(_sc) \ diff --git a/freebsd/sys/dev/rtwn/pci/rtwn_pci_attach.c b/freebsd/sys/dev/rtwn/pci/rtwn_pci_attach.c index 5b28d27f..9813cb32 100644 --- a/freebsd/sys/dev/rtwn/pci/rtwn_pci_attach.c +++ b/freebsd/sys/dev/rtwn/pci/rtwn_pci_attach.c @@ -96,20 +96,31 @@ static void rtwn_pci_beacon_update_end(struct rtwn_softc *, static void rtwn_pci_attach_methods(struct rtwn_softc *); -static int matched_chip = RTWN_CHIP_MAX_PCI; +static const struct rtwn_pci_ident * +rtwn_pci_probe_sub(device_t dev) +{ + const struct rtwn_pci_ident *ident; + int vendor_id, device_id; + + vendor_id = pci_get_vendor(dev); + device_id = pci_get_device(dev); + + for (ident = rtwn_pci_ident_table; ident->name != NULL; ident++) + if (vendor_id == ident->vendor && device_id == ident->device) + return (ident); + + return (NULL); +} static int rtwn_pci_probe(device_t dev) { const struct rtwn_pci_ident *ident; - for (ident = rtwn_pci_ident_table; ident->name != NULL; ident++) { - if (pci_get_vendor(dev) == ident->vendor && - pci_get_device(dev) == ident->device) { - matched_chip = ident->chip; - device_set_desc(dev, ident->name); - return (BUS_PROBE_DEFAULT); - } + ident = rtwn_pci_probe_sub(dev); + if (ident != NULL) { + device_set_desc(dev, ident->name); + return (BUS_PROBE_DEFAULT); } return (ENXIO); } @@ -593,13 +604,15 @@ rtwn_pci_attach_methods(struct rtwn_softc *sc) static int rtwn_pci_attach(device_t dev) { + const struct rtwn_pci_ident *ident; struct rtwn_pci_softc *pc = device_get_softc(dev); struct rtwn_softc *sc = &pc->pc_sc; struct ieee80211com *ic = &sc->sc_ic; uint32_t lcsr; int cap_off, i, error, rid; - if (matched_chip >= RTWN_CHIP_MAX_PCI) + ident = rtwn_pci_probe_sub(dev); + if (ident == NULL) return (ENXIO); /* @@ -651,8 +664,7 @@ rtwn_pci_attach(device_t dev) mtx_init(&sc->sc_mtx, ic->ic_name, MTX_NETWORK_LOCK, MTX_DEF); rtwn_pci_attach_methods(sc); - /* XXX something similar to USB_GET_DRIVER_INFO() */ - rtwn_pci_attach_private(pc, matched_chip); + rtwn_pci_attach_private(pc, ident->chip); /* Allocate Tx/Rx buffers. */ error = rtwn_pci_alloc_rx_list(sc); diff --git a/freebsd/sys/dev/rtwn/pci/rtwn_pci_rx.c b/freebsd/sys/dev/rtwn/pci/rtwn_pci_rx.c index 8da0061b..292fb07f 100644 --- a/freebsd/sys/dev/rtwn/pci/rtwn_pci_rx.c +++ b/freebsd/sys/dev/rtwn/pci/rtwn_pci_rx.c @@ -97,7 +97,6 @@ rtwn_pci_rx_frame(struct rtwn_softc *sc, struct r92ce_rx_stat *rx_desc, struct ieee80211_node *ni; uint32_t rxdw0; struct mbuf *m, *m1; - int8_t rssi = 0, nf; int infosz, pktlen, shift, error; /* Dump Rx descriptor. */ @@ -164,12 +163,11 @@ rtwn_pci_rx_frame(struct rtwn_softc *sc, struct r92ce_rx_stat *rx_desc, rx_data->m = m1; m->m_pkthdr.len = m->m_len = pktlen + infosz + shift; - nf = RTWN_NOISE_FLOOR; - ni = rtwn_rx_common(sc, m, rx_desc, &rssi); + ni = rtwn_rx_common(sc, m, rx_desc); RTWN_DPRINTF(sc, RTWN_DEBUG_RECV, - "%s: Rx frame len %d, infosz %d, shift %d, rssi %d\n", - __func__, pktlen, infosz, shift, rssi); + "%s: Rx frame len %d, infosz %d, shift %d\n", + __func__, pktlen, infosz, shift); /* Update RX descriptor. */ rtwn_pci_setup_rx_desc(pc, rx_desc, rx_data->paddr, MJUMPAGESIZE, @@ -178,11 +176,11 @@ rtwn_pci_rx_frame(struct rtwn_softc *sc, struct r92ce_rx_stat *rx_desc, /* Send the frame to the 802.11 layer. */ RTWN_UNLOCK(sc); if (ni != NULL) { - (void)ieee80211_input(ni, m, rssi - nf, nf); + (void)ieee80211_input_mimo(ni, m); /* Node is no longer needed. */ ieee80211_free_node(ni); } else - (void)ieee80211_input_all(ic, m, rssi - nf, nf); + (void)ieee80211_input_mimo_all(ic, m); RTWN_LOCK(sc); @@ -284,17 +282,6 @@ rtwn_pci_rx_done(struct rtwn_softc *sc) ring->cur = (ring->cur + 1) % RTWN_PCI_RX_LIST_COUNT; } - - /* Finished receive; age anything left on the FF queue by a little bump */ - /* - * XXX TODO: just make this a callout timer schedule so we can - * flush the FF staging queue if we're approaching idle. - */ -#ifdef IEEE80211_SUPPORT_SUPERG - if (!(sc->sc_flags & RTWN_FW_LOADED) || - sc->sc_ratectl != RTWN_RATECTL_NET80211) - rtwn_cmd_sleepable(sc, NULL, 0, rtwn_ff_flush_all); -#endif } void diff --git a/freebsd/sys/dev/rtwn/rtl8188e/r88e.h b/freebsd/sys/dev/rtwn/rtl8188e/r88e.h index 999ab400..1c03ddd3 100644 --- a/freebsd/sys/dev/rtwn/rtl8188e/r88e.h +++ b/freebsd/sys/dev/rtwn/rtl8188e/r88e.h @@ -85,6 +85,8 @@ void r88e_ratectl_tx_complete(struct rtwn_softc *, uint8_t *, int); void r88e_handle_c2h_report(struct rtwn_softc *, uint8_t *, int); int8_t r88e_get_rssi_cck(struct rtwn_softc *, void *); int8_t r88e_get_rssi_ofdm(struct rtwn_softc *, void *); +void r88e_get_rx_stats(struct rtwn_softc *, struct ieee80211_rx_stats *, + const void *, const void *); /* r88e_tx.c */ void r88e_tx_enable_ampdu(void *, int); diff --git a/freebsd/sys/dev/rtwn/rtl8188e/r88e_fw.c b/freebsd/sys/dev/rtwn/rtl8188e/r88e_fw.c index 409084f6..fb7743ed 100644 --- a/freebsd/sys/dev/rtwn/rtl8188e/r88e_fw.c +++ b/freebsd/sys/dev/rtwn/rtl8188e/r88e_fw.c @@ -71,7 +71,7 @@ r88e_fw_cmd(struct rtwn_softc *sc, uint8_t id, const void *buf, int len) } /* Wait for current FW box to be empty. */ - for (ntries = 0; ntries < 50; ntries++) { + for (ntries = 0; ntries < 100; ntries++) { if (!(rtwn_read_1(sc, R92C_HMETFR) & (1 << sc->fwcur))) break; rtwn_delay(sc, 2000); diff --git a/freebsd/sys/dev/rtwn/rtl8188e/r88e_rx.c b/freebsd/sys/dev/rtwn/rtl8188e/r88e_rx.c index 464542b4..acffb40e 100644 --- a/freebsd/sys/dev/rtwn/rtl8188e/r88e_rx.c +++ b/freebsd/sys/dev/rtwn/rtl8188e/r88e_rx.c @@ -53,6 +53,7 @@ __FBSDID("$FreeBSD$"); #include <dev/rtwn/if_rtwn_debug.h> #include <dev/rtwn/if_rtwn_ridx.h> +#include <dev/rtwn/rtl8192c/r92c.h> #include <dev/rtwn/rtl8188e/r88e.h> #include <dev/rtwn/rtl8188e/r88e_rx_desc.h> @@ -211,3 +212,19 @@ r88e_get_rssi_ofdm(struct rtwn_softc *sc, void *physt) return (rssi); } + +void +r88e_get_rx_stats(struct rtwn_softc *sc, struct ieee80211_rx_stats *rxs, + const void *desc, const void *physt_ptr) +{ + const struct r88e_rx_phystat *physt = physt_ptr; + + r92c_get_rx_stats(sc, rxs, desc, physt_ptr); + + if (!sc->sc_ht40) { /* XXX center channel */ + rxs->r_flags |= IEEE80211_R_IEEE | IEEE80211_R_FREQ; + rxs->c_ieee = le16toh(physt->chan); + rxs->c_freq = ieee80211_ieee2mhz(rxs->c_ieee, + IEEE80211_CHAN_2GHZ); + } +} diff --git a/freebsd/sys/dev/rtwn/rtl8188e/usb/r88eu_attach.c b/freebsd/sys/dev/rtwn/rtl8188e/usb/r88eu_attach.c index 4d5452be..f834fb38 100644 --- a/freebsd/sys/dev/rtwn/rtl8188e/usb/r88eu_attach.c +++ b/freebsd/sys/dev/rtwn/rtl8188e/usb/r88eu_attach.c @@ -129,6 +129,7 @@ r88eu_attach(struct rtwn_usb_softc *uc) sc->sc_dump_tx_desc = r92cu_dump_tx_desc; sc->sc_tx_radiotap_flags = r92c_tx_radiotap_flags; sc->sc_rx_radiotap_flags = r92c_rx_radiotap_flags; + sc->sc_get_rx_stats = r88e_get_rx_stats; sc->sc_get_rssi_cck = r88e_get_rssi_cck; sc->sc_get_rssi_ofdm = r88e_get_rssi_ofdm; sc->sc_classify_intr = r88eu_classify_intr; @@ -147,6 +148,7 @@ r88eu_attach(struct rtwn_usb_softc *uc) sc->sc_fw_reset = r88e_fw_reset; sc->sc_fw_download_enable = r88e_fw_download_enable; #endif + sc->sc_llt_init = r92c_llt_init; sc->sc_set_page_size = r92c_set_page_size; sc->sc_lc_calib = r92c_lc_calib; sc->sc_iq_calib = r88e_iq_calib; /* XXX TODO */ diff --git a/freebsd/sys/dev/rtwn/rtl8192c/pci/r92ce_attach.c b/freebsd/sys/dev/rtwn/rtl8192c/pci/r92ce_attach.c index d53dbf98..225c69f5 100644 --- a/freebsd/sys/dev/rtwn/rtl8192c/pci/r92ce_attach.c +++ b/freebsd/sys/dev/rtwn/rtl8192c/pci/r92ce_attach.c @@ -174,6 +174,7 @@ r92ce_attach(struct rtwn_pci_softc *pc) sc->sc_dump_tx_desc = r92ce_dump_tx_desc; sc->sc_tx_radiotap_flags = r92c_tx_radiotap_flags; sc->sc_rx_radiotap_flags = r92c_rx_radiotap_flags; + sc->sc_get_rx_stats = r92c_get_rx_stats; sc->sc_get_rssi_cck = r92c_get_rssi_cck; sc->sc_get_rssi_ofdm = r92c_get_rssi_ofdm; sc->sc_classify_intr = r92ce_classify_intr; @@ -192,6 +193,7 @@ r92ce_attach(struct rtwn_pci_softc *pc) sc->sc_fw_reset = r92ce_fw_reset; sc->sc_fw_download_enable = r92c_fw_download_enable; #endif + sc->sc_llt_init = r92c_llt_init; sc->sc_set_page_size = r92c_set_page_size; sc->sc_lc_calib = r92c_lc_calib; sc->sc_iq_calib = r92ce_iq_calib; diff --git a/freebsd/sys/dev/rtwn/rtl8192c/r92c.h b/freebsd/sys/dev/rtwn/rtl8192c/r92c.h index 2b63179e..5ac666d0 100644 --- a/freebsd/sys/dev/rtwn/rtl8192c/r92c.h +++ b/freebsd/sys/dev/rtwn/rtl8192c/r92c.h @@ -77,6 +77,7 @@ void r92c_handle_c2h_report(void *); /* r92c_init.c */ int r92c_check_condition(struct rtwn_softc *, const uint8_t[]); +int r92c_llt_init(struct rtwn_softc *); int r92c_set_page_size(struct rtwn_softc *); void r92c_init_bb_common(struct rtwn_softc *); int r92c_init_rf_chain(struct rtwn_softc *, @@ -87,6 +88,9 @@ void r92c_init_ampdu(struct rtwn_softc *); void r92c_init_antsel(struct rtwn_softc *); void r92c_pa_bias_init(struct rtwn_softc *); +/* r92c_llt.c */ +int r92c_llt_write(struct rtwn_softc *, uint32_t, uint32_t); + /* r92c_rf.c */ uint32_t r92c_rf_read(struct rtwn_softc *, int, uint8_t); void r92c_rf_write(struct rtwn_softc *, int, uint8_t, uint32_t); @@ -99,6 +103,8 @@ void r92c_parse_rom(struct rtwn_softc *, uint8_t *); int8_t r92c_get_rssi_cck(struct rtwn_softc *, void *); int8_t r92c_get_rssi_ofdm(struct rtwn_softc *, void *); uint8_t r92c_rx_radiotap_flags(const void *); +void r92c_get_rx_stats(struct rtwn_softc *, struct ieee80211_rx_stats *, + const void *, const void *); /* r92c_tx.c */ void r92c_tx_enable_ampdu(void *, int); diff --git a/freebsd/sys/dev/rtwn/rtl8192c/r92c_fw.c b/freebsd/sys/dev/rtwn/rtl8192c/r92c_fw.c index 74c7d205..91bcfc0e 100644 --- a/freebsd/sys/dev/rtwn/rtl8192c/r92c_fw.c +++ b/freebsd/sys/dev/rtwn/rtl8192c/r92c_fw.c @@ -82,7 +82,7 @@ r92c_fw_cmd(struct rtwn_softc *sc, uint8_t id, const void *buf, int len) } /* Wait for current FW box to be empty. */ - for (ntries = 0; ntries < 50; ntries++) { + for (ntries = 0; ntries < 100; ntries++) { if (!(rtwn_read_1(sc, R92C_HMETFR) & (1 << sc->fwcur))) break; rtwn_delay(sc, 2000); diff --git a/freebsd/sys/dev/rtwn/rtl8192c/r92c_init.c b/freebsd/sys/dev/rtwn/rtl8192c/r92c_init.c index d8db0286..4ec44045 100644 --- a/freebsd/sys/dev/rtwn/rtl8192c/r92c_init.c +++ b/freebsd/sys/dev/rtwn/rtl8192c/r92c_init.c @@ -92,6 +92,32 @@ r92c_check_condition(struct rtwn_softc *sc, const uint8_t cond[]) } int +r92c_llt_init(struct rtwn_softc *sc) +{ + int i, error; + + /* Reserve pages [0; page_count]. */ + for (i = 0; i < sc->page_count; i++) { + if ((error = r92c_llt_write(sc, i, i + 1)) != 0) + return (error); + } + /* NB: 0xff indicates end-of-list. */ + if ((error = r92c_llt_write(sc, i, 0xff)) != 0) + return (error); + /* + * Use pages [page_count + 1; pktbuf_count - 1] + * as ring buffer. + */ + for (++i; i < sc->pktbuf_count - 1; i++) { + if ((error = r92c_llt_write(sc, i, i + 1)) != 0) + return (error); + } + /* Make the last page point to the beginning of the ring buffer. */ + error = r92c_llt_write(sc, i, sc->page_count + 1); + return (error); +} + +int r92c_set_page_size(struct rtwn_softc *sc) { return (rtwn_write_1(sc, R92C_PBP, SM(R92C_PBP_PSRX, R92C_PBP_128) | diff --git a/freebsd/sys/dev/rtwn/rtl8192c/r92c_reg.h b/freebsd/sys/dev/rtwn/rtl8192c/r92c_reg.h index ff03d191..34a4b80c 100644 --- a/freebsd/sys/dev/rtwn/rtl8192c/r92c_reg.h +++ b/freebsd/sys/dev/rtwn/rtl8192c/r92c_reg.h @@ -66,6 +66,7 @@ #define R92C_HSIMR 0x058 #define R92C_HSISR 0x05c #define R92C_MULTI_FUNC_CTRL 0x068 +#define R92C_LDO_SWR_CTRL 0x07c #define R92C_MCUFWDL 0x080 #define R92C_HMEBOX_EXT(idx) (0x088 + (idx) * 2) #define R92C_EFUSE_ACCESS 0x0cf @@ -115,6 +116,7 @@ #define R92C_TXDMA_OFFSET_CHK 0x20c #define R92C_TXDMA_STATUS 0x210 #define R92C_RQPN_NPQ 0x214 +#define R92C_AUTO_LLT 0x224 /* Rx DMA Configuration. */ #define R92C_RXDMA_AGG_PG_TH 0x280 #define R92C_RXPKT_NUM 0x284 @@ -297,6 +299,16 @@ #define R92C_SYS_CLKR_SYS_EN 0x00001000 #define R92C_SYS_CLKR_RING_EN 0x00002000 +/* Bits for R92C_RSV_CTRL. */ +#define R92C_RSV_CTRL_WLOCK_ALL 0x01 +#define R92C_RSV_CTRL_WLOCK_00 0x02 +#define R92C_RSV_CTRL_WLOCK_04 0x04 +#define R92C_RSV_CTRL_WLOCK_08 0x08 +#define R92C_RSV_CTRL_WLOCK_40 0x10 +#define R92C_RSV_CTRL_R_DIS_PRST_0 0x20 +#define R92C_RSV_CTRL_R_DIS_PRST_1 0x40 +#define R92C_RSV_CTRL_LOCK_ALL_EN 0x80 + /* Bits for R92C_RF_CTRL. */ #define R92C_RF_CTRL_EN 0x01 #define R92C_RF_CTRL_RSTB 0x02 @@ -339,6 +351,9 @@ /* Bits for R92C_LEDCFG0. */ #define R92C_LEDCFG0_DIS 0x08 +/* Bits for R92C_LEDCFG1. */ +#define R92C_LEDCFG1_DIS 0x80 + /* Bits for R92C_MULTI_FUNC_CTRL. */ #define R92C_MULTI_BT_FUNC_EN 0x00040000 @@ -420,6 +435,7 @@ #define R92C_PBP_1024 4 /* Bits for R92C_TRXDMA_CTRL. */ +#define R92C_TRXDMA_CTRL_RX_SHIFT_EN 0x0002 #define R92C_TRXDMA_CTRL_RXDMA_AGG_EN 0x0004 #define R92C_TRXDMA_CTRL_TXDMA_VOQ_MAP_M 0x0030 #define R92C_TRXDMA_CTRL_TXDMA_VOQ_MAP_S 4 @@ -476,6 +492,9 @@ /* Bits for R92C_TXDMA_OFFSET_CHK. */ #define R92C_TXDMA_OFFSET_DROP_DATA_EN 0x00000200 +/* Bits for R92C_AUTO_LLT. */ +#define R92C_AUTO_LLT_INIT 0x00010000 + /* Bits for R92C_FWHW_TXQ_CTRL. */ #define R92C_FWHW_TXQ_CTRL_AMPDU_RTY_NEW 0x80 #define R92C_FWHW_TXQ_CTRL_REAL_BEACON 0x400000 @@ -593,7 +612,8 @@ #define R92C_RCR_APPFCS 0x80000000 /* Bits for R92C_RX_DRVINFO_SZ. */ -#define R92C_RX_DRVINFO_SZ_DEF 4 /* XXX other values will not work */ +/* XXX other values will not work */ +#define R92C_RX_DRVINFO_SZ_DEF ((RTWN_PHY_STATUS_SIZE) / 8) /* Bits for R92C_WMAC_TRXPTCL_CTL. */ #define R92C_WMAC_TRXPTCL_SHPRE 0x00020000 @@ -681,6 +701,7 @@ #define R92C_OFDM0_TXIQIMBALANCE(chain) (0xc80 + (chain) * 8) #define R92C_OFDM0_TXAFE(chain) (0xc94 + (chain) * 8) #define R92C_OFDM0_RXIQEXTANTA 0xca0 +#define R92C_OFDM0_TXPSEUDONOISEWGT 0xce4 #define R92C_OFDM1_LSTF 0xd00 /* Bits for R92C_FPGA[01]_RFMOD. */ @@ -800,6 +821,9 @@ #define R92C_LSSI_READBACK_DATA_M 0x000fffff #define R92C_LSSI_READBACK_DATA_S 0 +/* Bits for R92C_CCK0_SYSTEM. */ +#define R92C_CCK0_SYSTEM_CCK_SIDEBAND 0x00000010 + /* Bits for R92C_OFDM0_AGCCORE1(i). */ #define R92C_OFDM0_AGCCORE1_GAIN_M 0x0000007f #define R92C_OFDM0_AGCCORE1_GAIN_S 0 diff --git a/freebsd/sys/dev/rtwn/rtl8192c/r92c_rx.c b/freebsd/sys/dev/rtwn/rtl8192c/r92c_rx.c index b77c76f6..70dff0f6 100644 --- a/freebsd/sys/dev/rtwn/rtl8192c/r92c_rx.c +++ b/freebsd/sys/dev/rtwn/rtl8192c/r92c_rx.c @@ -102,3 +102,47 @@ r92c_rx_radiotap_flags(const void *buf) flags = IEEE80211_RADIOTAP_F_SHORTGI; return (flags); } + +void +r92c_get_rx_stats(struct rtwn_softc *sc, struct ieee80211_rx_stats *rxs, + const void *desc, const void *physt_ptr) +{ + const struct r92c_rx_stat *stat = desc; + uint32_t rxdw1, rxdw3; + uint8_t rate; + + rxdw1 = le32toh(stat->rxdw1); + rxdw3 = le32toh(stat->rxdw3); + rate = MS(rxdw3, R92C_RXDW3_RATE); + + if (rxdw1 & R92C_RXDW1_AMPDU) + rxs->c_pktflags |= IEEE80211_RX_F_AMPDU; + else if (rxdw1 & R92C_RXDW1_AMPDU_MORE) + rxs->c_pktflags |= IEEE80211_RX_F_AMPDU_MORE; + if ((rxdw3 & R92C_RXDW3_SPLCP) && rate >= RTWN_RIDX_MCS(0)) + rxs->c_pktflags |= IEEE80211_RX_F_SHORTGI; + + if (rxdw3 & R92C_RXDW3_HT40) + rxs->c_width = IEEE80211_RX_FW_40MHZ; + else + rxs->c_width = IEEE80211_RX_FW_20MHZ; + + if (RTWN_RATE_IS_CCK(rate)) + rxs->c_phytype = IEEE80211_RX_FP_11B; + else if (rate < RTWN_RIDX_MCS(0)) + rxs->c_phytype = IEEE80211_RX_FP_11G; + else + rxs->c_phytype = IEEE80211_RX_FP_11NG; + + /* Map HW rate index to 802.11 rate. */ + if (rate < RTWN_RIDX_MCS(0)) { + rxs->c_rate = ridx2rate[rate]; + if (RTWN_RATE_IS_CCK(rate)) + rxs->c_pktflags |= IEEE80211_RX_F_CCK; + else + rxs->c_pktflags |= IEEE80211_RX_F_OFDM; + } else { /* MCS0~15. */ + rxs->c_rate = IEEE80211_RATE_MCS | (rate - 12); + rxs->c_pktflags |= IEEE80211_RX_F_HT; + } +} diff --git a/freebsd/sys/dev/rtwn/rtl8192c/r92c_rx_desc.h b/freebsd/sys/dev/rtwn/rtl8192c/r92c_rx_desc.h index 7fec70be..12dfd665 100644 --- a/freebsd/sys/dev/rtwn/rtl8192c/r92c_rx_desc.h +++ b/freebsd/sys/dev/rtwn/rtl8192c/r92c_rx_desc.h @@ -45,6 +45,9 @@ struct r92c_rx_stat { uint32_t rxdw1; #define R92C_RXDW1_MACID_M 0x0000001f #define R92C_RXDW1_MACID_S 0 +#define R92C_RXDW1_AMSDU 0x00002000 +#define R92C_RXDW1_AMPDU_MORE 0x00004000 +#define R92C_RXDW1_AMPDU 0x00008000 #define R92C_RXDW1_MC 0x40000000 #define R92C_RXDW1_BC 0x80000000 @@ -56,6 +59,8 @@ struct r92c_rx_stat { #define R92C_RXDW3_SPLCP 0x00000100 #define R92C_RXDW3_HT40 0x00000200 #define R92C_RXDW3_HTC 0x00000400 +#define R92C_RXDW3_BSSID_FIT_M 0x00003000 +#define R92C_RXDW3_BSSID_FIT_S 12 uint32_t rxdw4; uint32_t tsf_low; diff --git a/freebsd/sys/dev/rtwn/rtl8192c/r92c_tx_desc.h b/freebsd/sys/dev/rtwn/rtl8192c/r92c_tx_desc.h index 037ac0e2..c3bc87ca 100644 --- a/freebsd/sys/dev/rtwn/rtl8192c/r92c_tx_desc.h +++ b/freebsd/sys/dev/rtwn/rtl8192c/r92c_tx_desc.h @@ -68,7 +68,7 @@ struct r92c_tx_desc { uint16_t txdseq; uint32_t txdw4; -#define R92C_TXDW4_RTSRATE_M 0x0000003f +#define R92C_TXDW4_RTSRATE_M 0x0000001f #define R92C_TXDW4_RTSRATE_S 0 #define R92C_TXDW4_SEQ_SEL_M 0x00000040 #define R92C_TXDW4_SEQ_SEL_S 6 diff --git a/freebsd/sys/dev/rtwn/rtl8192c/usb/r92cu_attach.c b/freebsd/sys/dev/rtwn/rtl8192c/usb/r92cu_attach.c index ce3f7a1a..aa6f7067 100644 --- a/freebsd/sys/dev/rtwn/rtl8192c/usb/r92cu_attach.c +++ b/freebsd/sys/dev/rtwn/rtl8192c/usb/r92cu_attach.c @@ -167,6 +167,7 @@ r92cu_attach(struct rtwn_usb_softc *uc) sc->sc_dump_tx_desc = r92cu_dump_tx_desc; sc->sc_tx_radiotap_flags = r92c_tx_radiotap_flags; sc->sc_rx_radiotap_flags = r92c_rx_radiotap_flags; + sc->sc_get_rx_stats = r92c_get_rx_stats; sc->sc_get_rssi_cck = r92c_get_rssi_cck; sc->sc_get_rssi_ofdm = r92c_get_rssi_ofdm; sc->sc_classify_intr = r92cu_classify_intr; @@ -185,6 +186,7 @@ r92cu_attach(struct rtwn_usb_softc *uc) sc->sc_fw_reset = r92c_fw_reset; sc->sc_fw_download_enable = r92c_fw_download_enable; #endif + sc->sc_llt_init = r92c_llt_init; sc->sc_set_page_size = r92c_set_page_size; sc->sc_lc_calib = r92c_lc_calib; sc->sc_iq_calib = r92c_iq_calib; /* XXX TODO */ diff --git a/freebsd/sys/dev/rtwn/rtl8812a/r12a.h b/freebsd/sys/dev/rtwn/rtl8812a/r12a.h index ec1d61e1..e8de45aa 100644 --- a/freebsd/sys/dev/rtwn/rtl8812a/r12a.h +++ b/freebsd/sys/dev/rtwn/rtl8812a/r12a.h @@ -128,6 +128,8 @@ void r12a_ratectl_tx_complete(struct rtwn_softc *, uint8_t *, int); void r12a_handle_c2h_report(struct rtwn_softc *, uint8_t *, int); int r12a_check_frame_checksum(struct rtwn_softc *, struct mbuf *); uint8_t r12a_rx_radiotap_flags(const void *); +void r12a_get_rx_stats(struct rtwn_softc *, struct ieee80211_rx_stats *, + const void *, const void *); /* r12a_tx.c */ void r12a_fill_tx_desc(struct rtwn_softc *, struct ieee80211_node *, diff --git a/freebsd/sys/dev/rtwn/rtl8812a/r12a_beacon.c b/freebsd/sys/dev/rtwn/rtl8812a/r12a_beacon.c index 37b1a183..67714442 100644 --- a/freebsd/sys/dev/rtwn/rtl8812a/r12a_beacon.c +++ b/freebsd/sys/dev/rtwn/rtl8812a/r12a_beacon.c @@ -79,6 +79,8 @@ r12a_beacon_init(struct rtwn_softc *sc, void *buf, int id) txd->txdw3 = htole32(R12A_TXDW3_DRVRATE); txd->txdw3 |= htole32(SM(R12A_TXDW3_SEQ_SEL, id)); + txd->txdw4 = htole32(SM(R12A_TXDW4_DATARATE, RTWN_RIDX_CCK1)); + txd->txdw6 = htole32(SM(R21A_TXDW6_MBSSID, id)); } diff --git a/freebsd/sys/dev/rtwn/rtl8812a/r12a_fw.c b/freebsd/sys/dev/rtwn/rtl8812a/r12a_fw.c index 12a3d855..f3bbc099 100644 --- a/freebsd/sys/dev/rtwn/rtl8812a/r12a_fw.c +++ b/freebsd/sys/dev/rtwn/rtl8812a/r12a_fw.c @@ -70,14 +70,14 @@ void r12a_fw_reset(struct rtwn_softc *sc, int reason) { /* Reset MCU IO wrapper. */ - rtwn_setbits_1(sc, R92C_RSV_CTRL, 0x02, 0); + rtwn_setbits_1(sc, R92C_RSV_CTRL, R92C_RSV_CTRL_WLOCK_00, 0); rtwn_setbits_1(sc, R92C_RSV_CTRL + 1, 0x08, 0); rtwn_setbits_1_shift(sc, R92C_SYS_FUNC_EN, R92C_SYS_FUNC_EN_CPUEN, 0, 1); /* Enable MCU IO wrapper. */ - rtwn_setbits_1(sc, R92C_RSV_CTRL, 0x02, 0); + rtwn_setbits_1(sc, R92C_RSV_CTRL, R92C_RSV_CTRL_WLOCK_00, 0); rtwn_setbits_1(sc, R92C_RSV_CTRL + 1, 0, 0x08); rtwn_setbits_1_shift(sc, R92C_SYS_FUNC_EN, diff --git a/freebsd/sys/dev/rtwn/rtl8812a/r12a_rx.c b/freebsd/sys/dev/rtwn/rtl8812a/r12a_rx.c index 049717a4..b9c3bbf8 100644 --- a/freebsd/sys/dev/rtwn/rtl8812a/r12a_rx.c +++ b/freebsd/sys/dev/rtwn/rtl8812a/r12a_rx.c @@ -231,10 +231,99 @@ r12a_rx_radiotap_flags(const void *buf) if (!(stat->rxdw4 & htole32(R12A_RXDW4_SPLCP))) return (0); - rate = MS(le32toh(stat->rxdw3), R92C_RXDW3_RATE); + rate = MS(le32toh(stat->rxdw3), R12A_RXDW3_RATE); if (RTWN_RATE_IS_CCK(rate)) flags = IEEE80211_RADIOTAP_F_SHORTPRE; else flags = IEEE80211_RADIOTAP_F_SHORTGI; return (flags); } + +void +r12a_get_rx_stats(struct rtwn_softc *sc, struct ieee80211_rx_stats *rxs, + const void *desc, const void *physt_ptr) +{ + const struct r92c_rx_stat *stat = desc; + const struct r12a_rx_phystat *physt = physt_ptr; + uint32_t rxdw0, rxdw1, rxdw3, rxdw4; + uint8_t rate; + + rxdw0 = le32toh(stat->rxdw0); + rxdw1 = le32toh(stat->rxdw1); + rxdw3 = le32toh(stat->rxdw3); + rxdw4 = le32toh(stat->rxdw4); + rate = MS(rxdw3, R12A_RXDW3_RATE); + + /* TODO: STBC */ + if (rxdw4 & R12A_RXDW4_LDPC) + rxs->c_pktflags |= IEEE80211_RX_F_LDPC; + if (rxdw1 & R12A_RXDW1_AMPDU) { + if (rxdw0 & R92C_RXDW0_PHYST) + rxs->c_pktflags |= IEEE80211_RX_F_AMPDU; + else + rxs->c_pktflags |= IEEE80211_RX_F_AMPDU_MORE; + } + + if ((rxdw4 & R12A_RXDW4_SPLCP) && rate >= RTWN_RIDX_MCS(0)) + rxs->c_pktflags |= IEEE80211_RX_F_SHORTGI; + + switch (MS(rxdw4, R12A_RXDW4_BW)) { + case R12A_RXDW4_BW20: + rxs->c_width = IEEE80211_RX_FW_20MHZ; + break; + case R12A_RXDW4_BW40: + rxs->c_width = IEEE80211_RX_FW_40MHZ; + break; + case R12A_RXDW4_BW80: + rxs->c_width = IEEE80211_RX_FW_80MHZ; + break; + default: + break; + } + + if (RTWN_RATE_IS_CCK(rate)) + rxs->c_phytype = IEEE80211_RX_FP_11B; + else { + int is5ghz; + + /* XXX magic */ + /* XXX check with RTL8812AU */ + is5ghz = (physt->cfosho[2] != 0x01); + + if (rate < RTWN_RIDX_MCS(0)) { + if (is5ghz) + rxs->c_phytype = IEEE80211_RX_FP_11A; + else + rxs->c_phytype = IEEE80211_RX_FP_11G; + } else { + if (is5ghz) + rxs->c_phytype = IEEE80211_RX_FP_11NA; + else + rxs->c_phytype = IEEE80211_RX_FP_11NG; + } + } + + /* Map HW rate index to 802.11 rate. */ + if (rate < RTWN_RIDX_MCS(0)) { + rxs->c_rate = ridx2rate[rate]; + if (RTWN_RATE_IS_CCK(rate)) + rxs->c_pktflags |= IEEE80211_RX_F_CCK; + else + rxs->c_pktflags |= IEEE80211_RX_F_OFDM; + } else { /* MCS0~15. */ + /* TODO: VHT rates */ + rxs->c_rate = IEEE80211_RATE_MCS | (rate - 12); + rxs->c_pktflags |= IEEE80211_RX_F_HT; + } + + /* + * XXX always zero for RTL8821AU + * (vendor driver does not check this field) + */ +#if 0 + rxs->r_flags |= IEEE80211_R_IEEE | IEEE80211_R_FREQ; + rxs->c_ieee = MS(le16toh(physt->phyw1), R12A_PHYW1_CHAN); + rxs->c_freq = ieee80211_ieee2mhz(rxs->c_ieee, + (rxs->c_ieee < 36) ? IEEE80211_CHAN_2GHZ : IEEE80211_CHAN_5GHZ); +#endif +} diff --git a/freebsd/sys/dev/rtwn/rtl8812a/r12a_rx_desc.h b/freebsd/sys/dev/rtwn/rtl8812a/r12a_rx_desc.h index 8642ca85..c3d19527 100644 --- a/freebsd/sys/dev/rtwn/rtl8812a/r12a_rx_desc.h +++ b/freebsd/sys/dev/rtwn/rtl8812a/r12a_rx_desc.h @@ -34,18 +34,26 @@ /* Rx MAC descriptor defines (chip-specific). */ /* Rx dword 1 */ #define R12A_RXDW1_AMSDU 0x00002000 +#define R12A_RXDW1_AMPDU 0x00008000 #define R12A_RXDW1_CKSUM_ERR 0x00100000 #define R12A_RXDW1_IPV6 0x00200000 #define R12A_RXDW1_UDP 0x00400000 #define R12A_RXDW1_CKSUM 0x00800000 /* Rx dword 2 */ #define R12A_RXDW2_RPT_C2H 0x10000000 +/* Rx dword 3 */ +#define R12A_RXDW3_RATE_M 0x0000007f +#define R12A_RXDW3_RATE_S 0 /* Rx dword 4 */ #define R12A_RXDW4_SPLCP 0x00000001 #define R12A_RXDW4_LDPC 0x00000002 #define R12A_RXDW4_STBC 0x00000004 #define R12A_RXDW4_BW_M 0x00000030 #define R12A_RXDW4_BW_S 4 +#define R12A_RXDW4_BW20 0 +#define R12A_RXDW4_BW40 1 +#define R12A_RXDW4_BW80 2 +#define R12A_RXDW4_BW160 3 /* Rx PHY descriptor. */ struct r12a_rx_phystat { diff --git a/freebsd/sys/dev/rtwn/rtl8812a/r12a_tx.c b/freebsd/sys/dev/rtwn/rtl8812a/r12a_tx.c index f7bd3a8e..40d54634 100644 --- a/freebsd/sys/dev/rtwn/rtl8812a/r12a_tx.c +++ b/freebsd/sys/dev/rtwn/rtl8812a/r12a_tx.c @@ -216,6 +216,17 @@ r12a_tx_set_sgi(struct rtwn_softc *sc, void *buf, struct ieee80211_node *ni) txd->txdw5 |= htole32(R12A_TXDW5_DATA_SHORT); } +static void +r12a_tx_set_ldpc(struct rtwn_softc *sc, struct r12a_tx_desc *txd, + struct ieee80211_node *ni) +{ + struct ieee80211vap *vap = ni->ni_vap; + + if ((vap->iv_flags_ht & IEEE80211_FHT_LDPC_TX) && + (ni->ni_htcap & IEEE80211_HTCAP_LDPC)) + txd->txdw5 |= htole32(R12A_TXDW5_DATA_LDPC); +} + void r12a_fill_tx_desc(struct rtwn_softc *sc, struct ieee80211_node *ni, struct mbuf *m, void *buf, uint8_t ridx, int maxretry) @@ -286,6 +297,7 @@ r12a_fill_tx_desc(struct rtwn_softc *sc, struct ieee80211_node *ni, if (ridx >= RTWN_RIDX_MCS(0)) { r12a_tx_set_ht40(sc, txd, ni); r12a_tx_set_sgi(sc, txd, ni); + r12a_tx_set_ldpc(sc, txd, ni); prot = ic->ic_htprotmode; } else if (ic->ic_flags & IEEE80211_F_USEPROT) prot = ic->ic_protmode; diff --git a/freebsd/sys/dev/rtwn/rtl8812a/usb/r12au_attach.c b/freebsd/sys/dev/rtwn/rtl8812a/usb/r12au_attach.c index 684076eb..97d966f0 100644 --- a/freebsd/sys/dev/rtwn/rtl8812a/usb/r12au_attach.c +++ b/freebsd/sys/dev/rtwn/rtl8812a/usb/r12au_attach.c @@ -170,7 +170,15 @@ r12a_read_chipid_vendor(struct rtwn_softc *sc, uint32_t reg_sys_cfg) static void r12au_adj_devcaps(struct rtwn_softc *sc) { - /* TODO: LDPC, STBC etc */ + struct r12a_softc *rs = sc->sc_priv; + struct ieee80211com *ic = &sc->sc_ic; + + if (rs->chip & R12A_CHIP_C_CUT) { + ic->ic_htcaps |= IEEE80211_HTCAP_LDPC | + IEEE80211_HTC_TXLDPC; + } + + /* TODO: STBC, VHT etc */ } void @@ -192,6 +200,7 @@ r12au_attach(struct rtwn_usb_softc *uc) sc->sc_dump_tx_desc = r12au_dump_tx_desc; sc->sc_tx_radiotap_flags = r12a_tx_radiotap_flags; sc->sc_rx_radiotap_flags = r12a_rx_radiotap_flags; + sc->sc_get_rx_stats = r12a_get_rx_stats; sc->sc_get_rssi_cck = r88e_get_rssi_cck; sc->sc_get_rssi_ofdm = r88e_get_rssi_ofdm; sc->sc_classify_intr = r12au_classify_intr; @@ -208,6 +217,7 @@ r12au_attach(struct rtwn_usb_softc *uc) sc->sc_fw_reset = r12a_fw_reset; sc->sc_fw_download_enable = r12a_fw_download_enable; #endif + sc->sc_llt_init = r92c_llt_init; sc->sc_set_page_size = r12a_set_page_size; sc->sc_lc_calib = r12a_lc_calib; sc->sc_iq_calib = r12a_iq_calib; diff --git a/freebsd/sys/dev/rtwn/rtl8821a/r21a_init.c b/freebsd/sys/dev/rtwn/rtl8821a/r21a_init.c index a3bcde77..e2c3972f 100644 --- a/freebsd/sys/dev/rtwn/rtl8821a/r21a_init.c +++ b/freebsd/sys/dev/rtwn/rtl8821a/r21a_init.c @@ -176,7 +176,7 @@ r21a_power_on(struct rtwn_softc *sc) R92C_CR_CALTMR_EN)); if (rtwn_read_4(sc, R92C_SYS_CFG) & R92C_SYS_CFG_TRP_BT_EN) - RTWN_CHK(rtwn_setbits_1(sc, 0x07C, 0, 0x40)); + RTWN_CHK(rtwn_setbits_1(sc, R92C_LDO_SWR_CTRL, 0, 0x40)); return (0); #undef RTWN_CHK diff --git a/freebsd/sys/dev/rtwn/rtl8821a/usb/r21au_attach.c b/freebsd/sys/dev/rtwn/rtl8821a/usb/r21au_attach.c index 6f7129f8..145aca21 100644 --- a/freebsd/sys/dev/rtwn/rtl8821a/usb/r21au_attach.c +++ b/freebsd/sys/dev/rtwn/rtl8821a/usb/r21au_attach.c @@ -159,10 +159,11 @@ r21au_adj_devcaps(struct rtwn_softc *sc) struct ieee80211com *ic = &sc->sc_ic; struct r12a_softc *rs = sc->sc_priv; + ic->ic_htcaps |= IEEE80211_HTC_TXLDPC; if (rs->rs_radar != 0) ic->ic_caps |= IEEE80211_C_DFS; - /* TODO: LDPC etc */ + /* TODO: VHT */ } void @@ -184,6 +185,7 @@ r21au_attach(struct rtwn_usb_softc *uc) sc->sc_dump_tx_desc = r12au_dump_tx_desc; sc->sc_tx_radiotap_flags = r12a_tx_radiotap_flags; sc->sc_rx_radiotap_flags = r12a_rx_radiotap_flags; + sc->sc_get_rx_stats = r12a_get_rx_stats; sc->sc_get_rssi_cck = r21a_get_rssi_cck; sc->sc_get_rssi_ofdm = r88e_get_rssi_ofdm; sc->sc_classify_intr = r12au_classify_intr; @@ -201,6 +203,7 @@ r21au_attach(struct rtwn_usb_softc *uc) sc->sc_fw_reset = r21a_fw_reset; sc->sc_fw_download_enable = r12a_fw_download_enable; #endif + sc->sc_llt_init = r92c_llt_init; sc->sc_set_page_size = rtwn_nop_int_softc; sc->sc_lc_calib = rtwn_nop_softc; /* XXX not used */ sc->sc_iq_calib = r12a_iq_calib; diff --git a/freebsd/sys/dev/rtwn/usb/rtwn_usb_attach.h b/freebsd/sys/dev/rtwn/usb/rtwn_usb_attach.h index 48a4d6e5..ee6d9137 100644 --- a/freebsd/sys/dev/rtwn/usb/rtwn_usb_attach.h +++ b/freebsd/sys/dev/rtwn/usb/rtwn_usb_attach.h @@ -21,12 +21,14 @@ */ void r92cu_attach(struct rtwn_usb_softc *); +void r92eu_attach(struct rtwn_usb_softc *); void r88eu_attach(struct rtwn_usb_softc *); void r12au_attach(struct rtwn_usb_softc *); void r21au_attach(struct rtwn_usb_softc *); enum { RTWN_CHIP_RTL8192CU, + RTWN_CHIP_RTL8192EU, RTWN_CHIP_RTL8188EU, RTWN_CHIP_RTL8812AU, RTWN_CHIP_RTL8821AU, @@ -92,7 +94,6 @@ static const STRUCT_USB_HOST_ID rtwn_devs[] = { RTWN_RTL8192CU_DEV(REALTEK, RTL8191CU), RTWN_RTL8192CU_DEV(REALTEK, RTL8192CE), RTWN_RTL8192CU_DEV(REALTEK, RTL8192CU), - RTWN_RTL8192CU_DEV(REALTEK, RTL8192CU_1), RTWN_RTL8192CU_DEV(SITECOMEU, RTL8188CU_1), RTWN_RTL8192CU_DEV(SITECOMEU, RTL8188CU_2), RTWN_RTL8192CU_DEV(SITECOMEU, RTL8192CU), @@ -101,6 +102,15 @@ static const STRUCT_USB_HOST_ID rtwn_devs[] = { RTWN_RTL8192CU_DEV(ZYXEL, RTL8192CU), #undef RTWN_RTL8192CU_DEV + /* RTL8192EU */ +#define RTWN_RTL8192EU_DEV(v,p) \ + { USB_VPI(USB_VENDOR_##v, USB_PRODUCT_##v##_##p, RTWN_CHIP_RTL8192EU) } + RTWN_RTL8192EU_DEV(DLINK, DWA131E1), + RTWN_RTL8192EU_DEV(REALTEK, RTL8192EU), + RTWN_RTL8192EU_DEV(TPLINK, WN822NV4), + RTWN_RTL8192EU_DEV(TPLINK, WN823NV2), +#undef RTWN_RTL8192EU_DEV + /* RTL8188EU */ #define RTWN_RTL8188EU_DEV(v,p) \ { USB_VPI(USB_VENDOR_##v, USB_PRODUCT_##v##_##p, RTWN_CHIP_RTL8188EU) } @@ -148,6 +158,7 @@ typedef void (*chip_usb_attach)(struct rtwn_usb_softc *); static const chip_usb_attach rtwn_chip_usb_attach[RTWN_CHIP_MAX_USB] = { [RTWN_CHIP_RTL8192CU] = r92cu_attach, + [RTWN_CHIP_RTL8192EU] = r92eu_attach, [RTWN_CHIP_RTL8188EU] = r88eu_attach, [RTWN_CHIP_RTL8812AU] = r12au_attach, [RTWN_CHIP_RTL8821AU] = r21au_attach diff --git a/freebsd/sys/dev/rtwn/usb/rtwn_usb_ep.c b/freebsd/sys/dev/rtwn/usb/rtwn_usb_ep.c index a1fafb46..b75aa3b9 100644 --- a/freebsd/sys/dev/rtwn/usb/rtwn_usb_ep.c +++ b/freebsd/sys/dev/rtwn/usb/rtwn_usb_ep.c @@ -60,7 +60,7 @@ __FBSDID("$FreeBSD$"); #include <dev/rtwn/rtl8192c/usb/r92cu_reg.h> -static struct usb_config rtwn_config[RTWN_N_TRANSFER] = { +static const struct usb_config rtwn_config_common[RTWN_N_TRANSFER] = { [RTWN_BULK_RX] = { .type = UE_BULK, .endpoint = UE_ADDR_ANY, @@ -163,6 +163,7 @@ rtwn_usb_setup_queues(struct rtwn_usb_softc *uc) int rtwn_usb_setup_endpoints(struct rtwn_usb_softc *uc) { + struct usb_config *rtwn_config; struct rtwn_softc *sc = &uc->uc_sc; const uint8_t iface_index = RTWN_IFACE_INDEX; struct usb_endpoint *ep, *ep_end; @@ -199,6 +200,9 @@ rtwn_usb_setup_endpoints(struct rtwn_usb_softc *uc) return (EINVAL); } + rtwn_config = malloc(sizeof(rtwn_config_common), M_TEMP, M_WAITOK); + memcpy(rtwn_config, rtwn_config_common, sizeof(rtwn_config_common)); + /* NB: keep in sync with rtwn_dma_init(). */ rtwn_config[RTWN_BULK_TX_VO].endpoint = addr[0]; switch (uc->ntx) { @@ -226,6 +230,8 @@ rtwn_usb_setup_endpoints(struct rtwn_usb_softc *uc) rtwn_config[RTWN_BULK_RX].bufsize = sc->rx_dma_size + 1024; error = usbd_transfer_setup(uc->uc_udev, &iface_index, uc->uc_xfer, rtwn_config, RTWN_N_TRANSFER, uc, &sc->sc_mtx); + free(rtwn_config, M_TEMP); + if (error) { device_printf(sc->sc_dev, "could not allocate USB transfers, " "err=%s\n", usbd_errstr(error)); diff --git a/freebsd/sys/dev/rtwn/usb/rtwn_usb_rx.c b/freebsd/sys/dev/rtwn/usb/rtwn_usb_rx.c index 8795e16d..4afa71af 100644 --- a/freebsd/sys/dev/rtwn/usb/rtwn_usb_rx.c +++ b/freebsd/sys/dev/rtwn/usb/rtwn_usb_rx.c @@ -238,7 +238,7 @@ rtwn_report_intr(struct rtwn_usb_softc *uc, struct usb_xfer *xfer, } static struct ieee80211_node * -rtwn_rx_frame(struct rtwn_softc *sc, struct mbuf *m, int8_t *rssi) +rtwn_rx_frame(struct rtwn_softc *sc, struct mbuf *m) { struct r92c_rx_stat stat; @@ -246,7 +246,7 @@ rtwn_rx_frame(struct rtwn_softc *sc, struct mbuf *m, int8_t *rssi) m_copydata(m, 0, sizeof(struct r92c_rx_stat), (caddr_t)&stat); m_adj(m, sizeof(struct r92c_rx_stat)); - return (rtwn_rx_common(sc, m, &stat, rssi)); + return (rtwn_rx_common(sc, m, &stat)); } void @@ -258,7 +258,6 @@ rtwn_bulk_rx_callback(struct usb_xfer *xfer, usb_error_t error) struct ieee80211_node *ni; struct mbuf *m = NULL, *next; struct rtwn_data *data; - int8_t nf, rssi; RTWN_ASSERT_LOCKED(sc); @@ -293,19 +292,15 @@ tr_setup: next = m->m_next; m->m_next = NULL; - ni = rtwn_rx_frame(sc, m, &rssi); + ni = rtwn_rx_frame(sc, m); RTWN_UNLOCK(sc); - nf = RTWN_NOISE_FLOOR; if (ni != NULL) { - if (ni->ni_flags & IEEE80211_NODE_HT) - m->m_flags |= M_AMPDU; - (void)ieee80211_input(ni, m, rssi - nf, nf); + (void)ieee80211_input_mimo(ni, m); ieee80211_free_node(ni); } else { - (void)ieee80211_input_all(ic, m, - rssi - nf, nf); + (void)ieee80211_input_mimo_all(ic, m); } RTWN_LOCK(sc); m = next; @@ -326,17 +321,6 @@ tr_setup: break; } finish: - /* Finished receive; age anything left on the FF queue by a little bump */ - /* - * XXX TODO: just make this a callout timer schedule so we can - * flush the FF staging queue if we're approaching idle. - */ -#ifdef IEEE80211_SUPPORT_SUPERG - if (!(sc->sc_flags & RTWN_FW_LOADED) || - sc->sc_ratectl != RTWN_RATECTL_NET80211) - rtwn_cmd_sleepable(sc, NULL, 0, rtwn_ff_flush_all); -#endif - /* Kick-start more transmit in case we stalled */ rtwn_start(sc); } diff --git a/freebsd/sys/dev/tsec/if_tsec.c b/freebsd/sys/dev/tsec/if_tsec.c index 5b94af9c..25802db0 100644 --- a/freebsd/sys/dev/tsec/if_tsec.c +++ b/freebsd/sys/dev/tsec/if_tsec.c @@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$"); #include <rtems/bsd/sys/param.h> #include <sys/systm.h> #include <sys/bus.h> +#include <sys/bus_dma.h> #include <sys/endian.h> #include <sys/mbuf.h> #include <sys/kernel.h> @@ -73,8 +74,8 @@ static int tsec_alloc_dma_desc(device_t dev, bus_dma_tag_t *dtag, bus_dmamap_t *dmap, bus_size_t dsize, void **vaddr, void *raddr, const char *dname); static void tsec_dma_ctl(struct tsec_softc *sc, int state); -static int tsec_encap(struct tsec_softc *sc, struct mbuf *m_head, - int fcb_inserted); +static void tsec_encap(struct ifnet *ifp, struct tsec_softc *sc, + struct mbuf *m0, uint16_t fcb_flags, int *start_tx); static void tsec_free_dma(struct tsec_softc *sc); static void tsec_free_dma_desc(bus_dma_tag_t dtag, bus_dmamap_t dmap, void *vaddr); static int tsec_ifmedia_upd(struct ifnet *ifp); @@ -123,8 +124,6 @@ tsec_attach(struct tsec_softc *sc) { uint8_t hwaddr[ETHER_ADDR_LEN]; struct ifnet *ifp; - bus_dmamap_t *map_ptr; - bus_dmamap_t **map_pptr; int error = 0; int i; @@ -181,7 +180,7 @@ tsec_attach(struct tsec_softc *sc) BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filtfunc, filtfuncarg */ MCLBYTES * (TSEC_TX_NUM_DESC - 1), /* maxsize */ - TSEC_TX_NUM_DESC - 1, /* nsegments */ + TSEC_TX_MAX_DMA_SEGS, /* nsegments */ MCLBYTES, 0, /* maxsegsz, flags */ NULL, NULL, /* lockfunc, lockfuncarg */ &sc->tsec_tx_mtag); /* dmat */ @@ -211,17 +210,15 @@ tsec_attach(struct tsec_softc *sc) } /* Create TX busdma maps */ - map_ptr = sc->tx_map_data; - map_pptr = sc->tx_map_unused_data; - for (i = 0; i < TSEC_TX_NUM_DESC; i++) { - map_pptr[i] = &map_ptr[i]; - error = bus_dmamap_create(sc->tsec_tx_mtag, 0, map_pptr[i]); + error = bus_dmamap_create(sc->tsec_tx_mtag, 0, + &sc->tx_bufmap[i].map); if (error) { device_printf(sc->dev, "failed to init TX ring\n"); tsec_detach(sc); return (ENXIO); } + sc->tx_bufmap[i].map_initialized = 1; } /* Create RX busdma maps and zero mbuf handlers */ @@ -370,13 +367,33 @@ tsec_init(void *xsc) TSEC_GLOBAL_UNLOCK(sc); } +static int +tsec_mii_wait(struct tsec_softc *sc, uint32_t flags) +{ + int timeout; + + /* + * The status indicators are not set immediatly after a command. + * Discard the first value. + */ + TSEC_PHY_READ(sc, TSEC_REG_MIIMIND); + + timeout = TSEC_READ_RETRY; + while ((TSEC_PHY_READ(sc, TSEC_REG_MIIMIND) & flags) && --timeout) + DELAY(TSEC_READ_DELAY); + + return (timeout == 0); +} + + static void tsec_init_locked(struct tsec_softc *sc) { struct tsec_desc *tx_desc = sc->tsec_tx_vaddr; struct tsec_desc *rx_desc = sc->tsec_rx_vaddr; struct ifnet *ifp = sc->tsec_ifp; - uint32_t timeout, val, i; + uint32_t val, i; + int timeout; if (ifp->if_drv_flags & IFF_DRV_RUNNING) return; @@ -435,15 +452,13 @@ tsec_init_locked(struct tsec_softc *sc) TSEC_PHY_WRITE(sc, TSEC_REG_MIIMCFG, TSEC_MIIMCFG_CLKDIV28); /* Step 8: Read MII Mgmt indicator register and check for Busy = 0 */ - timeout = TSEC_READ_RETRY; - while (--timeout && (TSEC_PHY_READ(sc, TSEC_REG_MIIMIND) & - TSEC_MIIMIND_BUSY)) - DELAY(TSEC_READ_DELAY); - if (timeout == 0) { + timeout = tsec_mii_wait(sc, TSEC_MIIMIND_BUSY); + + TSEC_PHY_UNLOCK(sc); + if (timeout) { if_printf(ifp, "tsec_init_locked(): Mgmt busy timeout\n"); return; } - TSEC_PHY_UNLOCK(sc); /* Step 9: Setup the MII Mgmt */ #ifdef __rtems__ @@ -724,124 +739,135 @@ static void tsec_start_locked(struct ifnet *ifp) { struct tsec_softc *sc; - struct mbuf *m0, *mtmp; + struct mbuf *m0; struct tsec_tx_fcb *tx_fcb; - unsigned int queued = 0; - int csum_flags, fcb_inserted = 0; + int csum_flags; + int start_tx; + uint16_t fcb_flags; sc = ifp->if_softc; + start_tx = 0; TSEC_TRANSMIT_LOCK_ASSERT(sc); - if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != - IFF_DRV_RUNNING) - return; - if (sc->tsec_link == 0) return; bus_dmamap_sync(sc->tsec_tx_dtag, sc->tsec_tx_dmap, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { + for (;;) { + + if (TSEC_FREE_TX_DESC(sc) < TSEC_TX_MAX_DMA_SEGS) { + /* No free descriptors */ + ifp->if_drv_flags |= IFF_DRV_OACTIVE; + break; + } + /* Get packet from the queue */ IFQ_DRV_DEQUEUE(&ifp->if_snd, m0); if (m0 == NULL) break; /* Insert TCP/IP Off-load frame control block */ + fcb_flags = 0; csum_flags = m0->m_pkthdr.csum_flags; if (csum_flags) { - M_PREPEND(m0, sizeof(struct tsec_tx_fcb), M_NOWAIT); if (m0 == NULL) break; - tx_fcb = mtod(m0, struct tsec_tx_fcb *); - tx_fcb->flags = 0; - tx_fcb->l3_offset = ETHER_HDR_LEN; - tx_fcb->l4_offset = sizeof(struct ip); - if (csum_flags & CSUM_IP) - tx_fcb->flags |= TSEC_TX_FCB_IP4 | + fcb_flags |= TSEC_TX_FCB_IP4 | TSEC_TX_FCB_CSUM_IP; if (csum_flags & CSUM_TCP) - tx_fcb->flags |= TSEC_TX_FCB_TCP | + fcb_flags |= TSEC_TX_FCB_TCP | TSEC_TX_FCB_CSUM_TCP_UDP; if (csum_flags & CSUM_UDP) - tx_fcb->flags |= TSEC_TX_FCB_UDP | + fcb_flags |= TSEC_TX_FCB_UDP | TSEC_TX_FCB_CSUM_TCP_UDP; - fcb_inserted = 1; + tx_fcb = mtod(m0, struct tsec_tx_fcb *); + tx_fcb->flags = fcb_flags; + tx_fcb->l3_offset = ETHER_HDR_LEN; + tx_fcb->l4_offset = sizeof(struct ip); } - mtmp = m_defrag(m0, M_NOWAIT); - if (mtmp) - m0 = mtmp; - - if (tsec_encap(sc, m0, fcb_inserted)) { - IFQ_DRV_PREPEND(&ifp->if_snd, m0); - ifp->if_drv_flags |= IFF_DRV_OACTIVE; - break; - } - queued++; - BPF_MTAP(ifp, m0); + tsec_encap(ifp, sc, m0, fcb_flags, &start_tx); } bus_dmamap_sync(sc->tsec_tx_dtag, sc->tsec_tx_dmap, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - if (queued) { + if (start_tx) { /* Enable transmitter and watchdog timer */ TSEC_WRITE(sc, TSEC_REG_TSTAT, TSEC_TSTAT_THLT); sc->tsec_watchdog = 5; } } -static int -tsec_encap(struct tsec_softc *sc, struct mbuf *m0, int fcb_inserted) +static void +tsec_encap(struct ifnet *ifp, struct tsec_softc *sc, struct mbuf *m0, + uint16_t fcb_flags, int *start_tx) { - struct tsec_desc *tx_desc = NULL; - struct ifnet *ifp; - bus_dma_segment_t segs[TSEC_TX_NUM_DESC]; - bus_dmamap_t *mapp; - int csum_flag = 0, error, seg, nsegs; + bus_dma_segment_t segs[TSEC_TX_MAX_DMA_SEGS]; + int error, i, nsegs; + struct tsec_bufmap *tx_bufmap; + uint32_t tx_idx; + uint16_t flags; TSEC_TRANSMIT_LOCK_ASSERT(sc); - ifp = sc->tsec_ifp; - - if (TSEC_FREE_TX_DESC(sc) == 0) { - /* No free descriptors */ - return (-1); - } - - /* Fetch unused map */ - mapp = TSEC_ALLOC_TX_MAP(sc); - + tx_idx = sc->tx_idx_head; + tx_bufmap = &sc->tx_bufmap[tx_idx]; + /* Create mapping in DMA memory */ - error = bus_dmamap_load_mbuf_sg(sc->tsec_tx_mtag, - *mapp, m0, segs, &nsegs, BUS_DMA_NOWAIT); - if (error != 0 || nsegs > TSEC_FREE_TX_DESC(sc) || nsegs <= 0) { - bus_dmamap_unload(sc->tsec_tx_mtag, *mapp); - TSEC_FREE_TX_MAP(sc, mapp); - return ((error != 0) ? error : -1); + error = bus_dmamap_load_mbuf_sg(sc->tsec_tx_mtag, tx_bufmap->map, m0, + segs, &nsegs, BUS_DMA_NOWAIT); + if (error == EFBIG) { + /* Too many segments! Defrag and try again. */ + struct mbuf *m = m_defrag(m0, M_NOWAIT); + + if (m == NULL) { + m_freem(m0); + return; + } + m0 = m; + error = bus_dmamap_load_mbuf_sg(sc->tsec_tx_mtag, + tx_bufmap->map, m0, segs, &nsegs, BUS_DMA_NOWAIT); + } + if (error != 0) { + /* Give up. */ + m_freem(m0); + return; } - bus_dmamap_sync(sc->tsec_tx_mtag, *mapp, BUS_DMASYNC_PREWRITE); - - if ((ifp->if_flags & IFF_DEBUG) && (nsegs > 1)) - if_printf(ifp, "TX buffer has %d segments\n", nsegs); - - if (fcb_inserted) - csum_flag = TSEC_TXBD_TOE; - - /* Everything is ok, now we can send buffers */ - for (seg = 0; seg < nsegs; seg++) { - tx_desc = TSEC_GET_CUR_TX_DESC(sc); - tx_desc->length = segs[seg].ds_len; - tx_desc->bufptr = segs[seg].ds_addr; + bus_dmamap_sync(sc->tsec_tx_mtag, tx_bufmap->map, + BUS_DMASYNC_PREWRITE); + tx_bufmap->mbuf = m0; + + /* + * Fill in the TX descriptors back to front so that READY bit in first + * descriptor is set last. + */ + tx_idx = (tx_idx + (uint32_t)nsegs) & (TSEC_TX_NUM_DESC - 1); + sc->tx_idx_head = tx_idx; + flags = TSEC_TXBD_L | TSEC_TXBD_I | TSEC_TXBD_R | TSEC_TXBD_TC; + for (i = nsegs - 1; i >= 0; i--) { + struct tsec_desc *tx_desc; + + tx_idx = (tx_idx - 1) & (TSEC_TX_NUM_DESC - 1); + tx_desc = &sc->tsec_tx_vaddr[tx_idx]; + tx_desc->length = segs[i].ds_len; + tx_desc->bufptr = segs[i].ds_addr; + + if (i == 0) { + wmb(); + + if (fcb_flags != 0) + flags |= TSEC_TXBD_TOE; + } /* * Set flags: @@ -851,17 +877,14 @@ tsec_encap(struct tsec_softc *sc, struct mbuf *m0, int fcb_inserted) * - transmit the CRC sequence after the last data byte * - interrupt after the last buffer */ - tx_desc->flags = - (tx_desc->flags & TSEC_TXBD_W) | - ((seg == 0) ? csum_flag : 0) | TSEC_TXBD_R | TSEC_TXBD_TC | - ((seg == nsegs - 1) ? TSEC_TXBD_L | TSEC_TXBD_I : 0); - } + tx_desc->flags = (tx_idx == (TSEC_TX_NUM_DESC - 1) ? + TSEC_TXBD_W : 0) | flags; - /* Save mbuf and DMA mapping for release at later stage */ - TSEC_PUT_TX_MBUF(sc, m0); - TSEC_PUT_TX_MAP(sc, mapp); + flags &= ~(TSEC_TXBD_L | TSEC_TXBD_I); + } - return (0); + BPF_MTAP(ifp, m0); + *start_tx = 1; } static void @@ -925,11 +948,8 @@ tsec_ioctl(struct ifnet *ifp, u_long command, caddr_t data) { struct tsec_softc *sc = ifp->if_softc; struct ifreq *ifr = (struct ifreq *)data; - device_t dev; int mask, error = 0; - dev = sc->dev; - switch (command) { case SIOCSIFMTU: TSEC_GLOBAL_LOCK(sc); @@ -1188,9 +1208,9 @@ tsec_free_dma(struct tsec_softc *sc) /* Free TX maps */ for (i = 0; i < TSEC_TX_NUM_DESC; i++) - if (sc->tx_map_data[i] != NULL) + if (sc->tx_bufmap[i].map_initialized) bus_dmamap_destroy(sc->tsec_tx_mtag, - sc->tx_map_data[i]); + sc->tx_bufmap[i].map); /* Destroy tag for TX mbufs */ bus_dma_tag_destroy(sc->tsec_tx_mtag); @@ -1225,8 +1245,6 @@ static void tsec_stop(struct tsec_softc *sc) { struct ifnet *ifp; - struct mbuf *m0; - bus_dmamap_t *mapp; uint32_t tmpval; TSEC_GLOBAL_LOCK_ASSERT(sc); @@ -1243,16 +1261,15 @@ tsec_stop(struct tsec_softc *sc) tsec_dma_ctl(sc, 0); /* Remove pending data from TX queue */ - while (!TSEC_EMPTYQ_TX_MBUF(sc)) { - m0 = TSEC_GET_TX_MBUF(sc); - mapp = TSEC_GET_TX_MAP(sc); - - bus_dmamap_sync(sc->tsec_tx_mtag, *mapp, + while (sc->tx_idx_tail != sc->tx_idx_head) { + bus_dmamap_sync(sc->tsec_tx_mtag, + sc->tx_bufmap[sc->tx_idx_tail].map, BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(sc->tsec_tx_mtag, *mapp); - - TSEC_FREE_TX_MAP(sc, mapp); - m_freem(m0); + bus_dmamap_unload(sc->tsec_tx_mtag, + sc->tx_bufmap[sc->tx_idx_tail].map); + m_freem(sc->tx_bufmap[sc->tx_idx_tail].mbuf); + sc->tx_idx_tail = (sc->tx_idx_tail + 1) + & (TSEC_TX_NUM_DESC - 1); } /* Disable RX and TX */ @@ -1304,7 +1321,6 @@ tsec_receive_intr_locked(struct tsec_softc *sc, int count) struct ifnet *ifp; struct rx_data_type *rx_data; struct mbuf *m; - device_t dev; uint32_t i; int c, rx_npkts; uint16_t flags; @@ -1313,7 +1329,6 @@ tsec_receive_intr_locked(struct tsec_softc *sc, int count) ifp = sc->tsec_ifp; rx_data = sc->rx_data; - dev = sc->dev; rx_npkts = 0; bus_dmamap_sync(sc->tsec_rx_dtag, sc->tsec_rx_dmap, @@ -1381,7 +1396,7 @@ tsec_receive_intr_locked(struct tsec_softc *sc, int count) if (tsec_new_rxbuf(sc->tsec_rx_mtag, rx_data[i].map, &rx_data[i].mbuf, &rx_data[i].paddr)) { - if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); + if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); /* * We ran out of mbufs; didn't consume current * descriptor and have to return it to the queue. @@ -1451,11 +1466,8 @@ tsec_receive_intr(void *arg) static void tsec_transmit_intr_locked(struct tsec_softc *sc) { - struct tsec_desc *tx_desc; struct ifnet *ifp; - struct mbuf *m0; - bus_dmamap_t *mapp; - int send = 0; + uint32_t tx_idx; TSEC_TRANSMIT_LOCK_ASSERT(sc); @@ -1474,44 +1486,41 @@ tsec_transmit_intr_locked(struct tsec_softc *sc) bus_dmamap_sync(sc->tsec_tx_dtag, sc->tsec_tx_dmap, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - while (TSEC_CUR_DIFF_DIRTY_TX_DESC(sc)) { - tx_desc = TSEC_GET_DIRTY_TX_DESC(sc); + tx_idx = sc->tx_idx_tail; + while (tx_idx != sc->tx_idx_head) { + struct tsec_desc *tx_desc; + struct tsec_bufmap *tx_bufmap; + + tx_desc = &sc->tsec_tx_vaddr[tx_idx]; if (tx_desc->flags & TSEC_TXBD_R) { - TSEC_BACK_DIRTY_TX_DESC(sc); break; } - if ((tx_desc->flags & TSEC_TXBD_L) == 0) + tx_bufmap = &sc->tx_bufmap[tx_idx]; + tx_idx = (tx_idx + 1) & (TSEC_TX_NUM_DESC - 1); + if (tx_bufmap->mbuf == NULL) continue; /* * This is the last buf in this packet, so unmap and free it. */ - m0 = TSEC_GET_TX_MBUF(sc); - mapp = TSEC_GET_TX_MAP(sc); - - bus_dmamap_sync(sc->tsec_tx_mtag, *mapp, + bus_dmamap_sync(sc->tsec_tx_mtag, tx_bufmap->map, BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(sc->tsec_tx_mtag, *mapp); - - TSEC_FREE_TX_MAP(sc, mapp); - m_freem(m0); + bus_dmamap_unload(sc->tsec_tx_mtag, tx_bufmap->map); + m_freem(tx_bufmap->mbuf); + tx_bufmap->mbuf = NULL; if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); - send = 1; } + sc->tx_idx_tail = tx_idx; bus_dmamap_sync(sc->tsec_tx_dtag, sc->tsec_tx_dmap, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - if (send) { - /* Now send anything that was pending */ - ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; - tsec_start_locked(ifp); + ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; + tsec_start_locked(ifp); - /* Stop wathdog if all sent */ - if (TSEC_EMPTYQ_TX_MBUF(sc)) - sc->tsec_watchdog = 0; - } + if (sc->tx_idx_tail == sc->tx_idx_head) + sc->tsec_watchdog = 0; } void @@ -1562,13 +1571,9 @@ tsec_error_intr_locked(struct tsec_softc *sc, int count) TSEC_WRITE(sc, TSEC_REG_TSTAT, TSEC_TSTAT_THLT); } - /* Check receiver errors */ + /* Check for discarded frame due to a lack of buffers */ if (eflags & TSEC_IEVENT_BSY) { - if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); - - /* Get data from RX buffers */ - tsec_receive_intr_locked(sc, count); } if (ifp->if_flags & IFF_DEBUG) @@ -1602,7 +1607,7 @@ int tsec_miibus_readreg(device_t dev, int phy, int reg) { struct tsec_softc *sc; - uint32_t timeout; + int timeout; int rv; sc = device_get_softc(dev); @@ -1612,17 +1617,13 @@ tsec_miibus_readreg(device_t dev, int phy, int reg) TSEC_PHY_WRITE(sc, TSEC_REG_MIIMCOM, 0); TSEC_PHY_WRITE(sc, TSEC_REG_MIIMCOM, TSEC_MIIMCOM_READCYCLE); - timeout = TSEC_READ_RETRY; - while (--timeout && TSEC_PHY_READ(sc, TSEC_REG_MIIMIND) & - (TSEC_MIIMIND_NOTVALID | TSEC_MIIMIND_BUSY)) - DELAY(TSEC_READ_DELAY); - - if (timeout == 0) - device_printf(dev, "Timeout while reading from PHY!\n"); - + timeout = tsec_mii_wait(sc, TSEC_MIIMIND_NOTVALID | TSEC_MIIMIND_BUSY); rv = TSEC_PHY_READ(sc, TSEC_REG_MIIMSTAT); TSEC_PHY_UNLOCK(); + if (timeout) + device_printf(dev, "Timeout while reading from PHY!\n"); + return (rv); } @@ -1630,21 +1631,17 @@ int tsec_miibus_writereg(device_t dev, int phy, int reg, int value) { struct tsec_softc *sc; - uint32_t timeout; + int timeout; sc = device_get_softc(dev); TSEC_PHY_LOCK(); TSEC_PHY_WRITE(sc, TSEC_REG_MIIMADD, (phy << 8) | reg); TSEC_PHY_WRITE(sc, TSEC_REG_MIIMCON, value); - - timeout = TSEC_READ_RETRY; - while (--timeout && (TSEC_READ(sc, TSEC_REG_MIIMIND) & - TSEC_MIIMIND_BUSY)) - DELAY(TSEC_READ_DELAY); + timeout = tsec_mii_wait(sc, TSEC_MIIMIND_BUSY); TSEC_PHY_UNLOCK(); - if (timeout == 0) + if (timeout) device_printf(dev, "Timeout while writing to PHY!\n"); return (0); diff --git a/freebsd/sys/dev/tsec/if_tsec.h b/freebsd/sys/dev/tsec/if_tsec.h index d13f4639..c8dca3bf 100644 --- a/freebsd/sys/dev/tsec/if_tsec.h +++ b/freebsd/sys/dev/tsec/if_tsec.h @@ -32,6 +32,7 @@ #define TSEC_RX_NUM_DESC 256 #define TSEC_TX_NUM_DESC 256 +#define TSEC_TX_MAX_DMA_SEGS 8 /* Interrupt Coalescing types */ #define TSEC_IC_RX 0 @@ -44,6 +45,12 @@ #define TSEC_MIN_FRAME_SIZE 64 #define TSEC_MAX_FRAME_SIZE 9600 +struct tsec_bufmap { + bus_dmamap_t map; + int map_initialized; + struct mbuf *mbuf; +}; + struct tsec_softc { /* XXX MII bus requires that struct ifnet is first!!! */ struct ifnet *tsec_ifp; @@ -59,16 +66,16 @@ struct tsec_softc { bus_dma_tag_t tsec_tx_dtag; /* TX descriptors tag */ bus_dmamap_t tsec_tx_dmap; /* TX descriptors map */ - struct tsec_desc *tsec_tx_vaddr;/* vadress of TX descriptors */ - uint32_t tsec_tx_raddr; /* real address of TX descriptors */ + bus_dma_tag_t tsec_tx_mtag; /* TX mbufs tag */ + uint32_t tx_idx_head; /* TX head descriptor/bufmap index */ + uint32_t tx_idx_tail; /* TX tail descriptor/bufmap index */ + struct tsec_desc *tsec_tx_vaddr;/* virtual address of TX descriptors */ + struct tsec_bufmap tx_bufmap[TSEC_TX_NUM_DESC]; + bus_dma_tag_t tsec_rx_mtag; /* TX mbufs tag */ bus_dma_tag_t tsec_rx_dtag; /* RX descriptors tag */ bus_dmamap_t tsec_rx_dmap; /* RX descriptors map */ struct tsec_desc *tsec_rx_vaddr; /* vadress of RX descriptors */ - uint32_t tsec_rx_raddr; /* real address of RX descriptors */ - - bus_dma_tag_t tsec_tx_mtag; /* TX mbufs tag */ - bus_dma_tag_t tsec_rx_mtag; /* TX mbufs tag */ struct rx_data_type { bus_dmamap_t map; /* mbuf map */ @@ -76,8 +83,6 @@ struct tsec_softc { uint32_t paddr; /* DMA address of buffer */ } rx_data[TSEC_RX_NUM_DESC]; - uint32_t tx_cur_desc_cnt; - uint32_t tx_dirty_desc_cnt; uint32_t rx_cur_desc_cnt; struct resource *sc_rres; /* register resource */ @@ -104,24 +109,6 @@ struct tsec_softc { struct callout tsec_callout; int tsec_watchdog; - /* TX maps */ - bus_dmamap_t tx_map_data[TSEC_TX_NUM_DESC]; - - /* unused TX maps data */ - uint32_t tx_map_unused_get_cnt; - uint32_t tx_map_unused_put_cnt; - bus_dmamap_t *tx_map_unused_data[TSEC_TX_NUM_DESC]; - - /* used TX maps data */ - uint32_t tx_map_used_get_cnt; - uint32_t tx_map_used_put_cnt; - bus_dmamap_t *tx_map_used_data[TSEC_TX_NUM_DESC]; - - /* mbufs in TX queue */ - uint32_t tx_mbuf_used_get_cnt; - uint32_t tx_mbuf_used_put_cnt; - struct mbuf *tx_mbuf_used_data[TSEC_TX_NUM_DESC]; - /* interrupt coalescing */ struct mtx ic_lock; uint32_t rx_ic_time; /* RW, valid values 0..65535 */ @@ -136,6 +123,9 @@ struct tsec_softc { bus_space_tag_t phy_bst; bus_space_handle_t phy_bsh; int phy_regoff; + + uint32_t tsec_rx_raddr; /* real address of RX descriptors */ + uint32_t tsec_tx_raddr; /* real address of TX descriptors */ }; /* interface to get/put generic objects */ @@ -156,75 +146,8 @@ struct tsec_softc { (sc)->count = (wrap) - 1; \ } while (0) -/* TX maps interface */ -#define TSEC_TX_MAP_CNT_INIT(sc) do { \ - TSEC_CNT_INIT((sc)->tx_map_unused_get_cnt, TSEC_TX_NUM_DESC); \ - TSEC_CNT_INIT((sc)->tx_map_unused_put_cnt, TSEC_TX_NUM_DESC); \ - TSEC_CNT_INIT((sc)->tx_map_used_get_cnt, TSEC_TX_NUM_DESC); \ - TSEC_CNT_INIT((sc)->tx_map_used_put_cnt, TSEC_TX_NUM_DESC); \ -} while (0) - -/* interface to get/put unused TX maps */ -#define TSEC_ALLOC_TX_MAP(sc) \ - TSEC_GET_GENERIC(sc, tx_map_unused_data, tx_map_unused_get_cnt, \ - TSEC_TX_NUM_DESC) - -#define TSEC_FREE_TX_MAP(sc, val) \ - TSEC_PUT_GENERIC(sc, tx_map_unused_data, tx_map_unused_put_cnt, \ - TSEC_TX_NUM_DESC, val) - -/* interface to get/put used TX maps */ -#define TSEC_GET_TX_MAP(sc) \ - TSEC_GET_GENERIC(sc, tx_map_used_data, tx_map_used_get_cnt, \ - TSEC_TX_NUM_DESC) - -#define TSEC_PUT_TX_MAP(sc, val) \ - TSEC_PUT_GENERIC(sc, tx_map_used_data, tx_map_used_put_cnt, \ - TSEC_TX_NUM_DESC, val) - -/* interface to get/put TX mbufs in send queue */ -#define TSEC_TX_MBUF_CNT_INIT(sc) do { \ - TSEC_CNT_INIT((sc)->tx_mbuf_used_get_cnt, TSEC_TX_NUM_DESC); \ - TSEC_CNT_INIT((sc)->tx_mbuf_used_put_cnt, TSEC_TX_NUM_DESC); \ -} while (0) - -#define TSEC_GET_TX_MBUF(sc) \ - TSEC_GET_GENERIC(sc, tx_mbuf_used_data, tx_mbuf_used_get_cnt, \ - TSEC_TX_NUM_DESC) - -#define TSEC_PUT_TX_MBUF(sc, val) \ - TSEC_PUT_GENERIC(sc, tx_mbuf_used_data, tx_mbuf_used_put_cnt, \ - TSEC_TX_NUM_DESC, val) - -#define TSEC_EMPTYQ_TX_MBUF(sc) \ - ((sc)->tx_mbuf_used_get_cnt == (sc)->tx_mbuf_used_put_cnt) - -/* interface for manage tx tsec_desc */ -#define TSEC_TX_DESC_CNT_INIT(sc) do { \ - TSEC_CNT_INIT((sc)->tx_cur_desc_cnt, TSEC_TX_NUM_DESC); \ - TSEC_CNT_INIT((sc)->tx_dirty_desc_cnt, TSEC_TX_NUM_DESC); \ -} while (0) - -#define TSEC_GET_CUR_TX_DESC(sc) \ - &TSEC_GET_GENERIC(sc, tsec_tx_vaddr, tx_cur_desc_cnt, \ - TSEC_TX_NUM_DESC) - -#define TSEC_GET_DIRTY_TX_DESC(sc) \ - &TSEC_GET_GENERIC(sc, tsec_tx_vaddr, tx_dirty_desc_cnt, \ - TSEC_TX_NUM_DESC) - -#define TSEC_BACK_DIRTY_TX_DESC(sc) \ - TSEC_BACK_GENERIC(sc, tx_dirty_desc_cnt, TSEC_TX_NUM_DESC) - -#define TSEC_CUR_DIFF_DIRTY_TX_DESC(sc) \ - ((sc)->tx_cur_desc_cnt != (sc)->tx_dirty_desc_cnt) - -#define TSEC_FREE_TX_DESC(sc) \ - (((sc)->tx_cur_desc_cnt < (sc)->tx_dirty_desc_cnt) ? \ - ((sc)->tx_dirty_desc_cnt - (sc)->tx_cur_desc_cnt - 1) \ - : \ - (TSEC_TX_NUM_DESC - (sc)->tx_cur_desc_cnt \ - + (sc)->tx_dirty_desc_cnt - 1)) +#define TSEC_FREE_TX_DESC(sc) \ + (((sc)->tx_idx_tail - (sc)->tx_idx_head - 1) & (TSEC_TX_NUM_DESC - 1)) /* interface for manage rx tsec_desc */ #define TSEC_RX_DESC_CNT_INIT(sc) do { \ @@ -243,9 +166,8 @@ struct tsec_softc { /* init all counters (for init only!) */ #define TSEC_TX_RX_COUNTERS_INIT(sc) do { \ - TSEC_TX_MAP_CNT_INIT(sc); \ - TSEC_TX_MBUF_CNT_INIT(sc); \ - TSEC_TX_DESC_CNT_INIT(sc); \ + sc->tx_idx_head = 0; \ + sc->tx_idx_tail = 0; \ TSEC_RX_DESC_CNT_INIT(sc); \ } while (0) diff --git a/freebsd/sys/dev/usb/quirk/usb_quirk.c b/freebsd/sys/dev/usb/quirk/usb_quirk.c index d73116e2..794bb0b0 100644 --- a/freebsd/sys/dev/usb/quirk/usb_quirk.c +++ b/freebsd/sys/dev/usb/quirk/usb_quirk.c @@ -521,6 +521,7 @@ static struct usb_quirk_entry usb_quirks[USB_DEV_QUIRKS_MAX] = { /* Non-standard USB AUDIO devices */ USB_QUIRK(MAUDIO, FASTTRACKULTRA, 0x0000, 0xffff, UQ_AU_VENDOR_CLASS), USB_QUIRK(MAUDIO, FASTTRACKULTRA8R, 0x0000, 0xffff, UQ_AU_VENDOR_CLASS), + USB_QUIRK(CMEDIA, CM6206, 0x0000, 0xffff, UQ_AU_SET_SPDIF_CM6206), /* * Quirks for manufacturers which USB devices does not respond @@ -607,6 +608,7 @@ static const char *usb_quirk_str[USB_QUIRK_MAX] = { [UQ_AU_VENDOR_CLASS] = "UQ_AU_VENDOR_CLASS", [UQ_SINGLE_CMD_MIDI] = "UQ_SINGLE_CMD_MIDI", [UQ_MSC_DYMO_EJECT] = "UQ_MSC_DYMO_EJECT", + [UQ_AU_SET_SPDIF_CM6206] = "UQ_AU_SET_SPDIF_CM6206", }; /*------------------------------------------------------------------------* diff --git a/freebsd/sys/dev/usb/quirk/usb_quirk.h b/freebsd/sys/dev/usb/quirk/usb_quirk.h index 7010916c..f7e490ce 100644 --- a/freebsd/sys/dev/usb/quirk/usb_quirk.h +++ b/freebsd/sys/dev/usb/quirk/usb_quirk.h @@ -109,6 +109,7 @@ enum { UQ_AU_VENDOR_CLASS, /* audio device uses vendor and not audio class */ UQ_SINGLE_CMD_MIDI, /* at most one command per USB packet */ UQ_MSC_DYMO_EJECT, /* ejects Dymo MSC device */ + UQ_AU_SET_SPDIF_CM6206, /* enable S/PDIF audio output */ USB_QUIRK_MAX }; diff --git a/freebsd/sys/dev/usb/usb_hub.c b/freebsd/sys/dev/usb/usb_hub.c index b7d5b597..b3ee8ab7 100644 --- a/freebsd/sys/dev/usb/usb_hub.c +++ b/freebsd/sys/dev/usb/usb_hub.c @@ -2300,7 +2300,7 @@ usb_needs_explore(struct usb_bus *bus, uint8_t do_probe) * usb_needs_explore_all * * This function is called whenever a new driver is loaded and will - * cause that all USB busses are re-explored. + * cause that all USB buses are re-explored. *------------------------------------------------------------------------*/ void usb_needs_explore_all(void) @@ -2318,7 +2318,7 @@ usb_needs_explore_all(void) return; } /* - * Explore all USB busses in parallel. + * Explore all USB buses in parallel. */ max = devclass_get_maxunit(dc); while (max >= 0) { diff --git a/freebsd/sys/dev/usb/usb_pf.h b/freebsd/sys/dev/usb/usb_pf.h index 9d51e98c..29fe6ebd 100644 --- a/freebsd/sys/dev/usb/usb_pf.h +++ b/freebsd/sys/dev/usb/usb_pf.h @@ -15,7 +15,7 @@ * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. - * 4. Neither the name of the University nor the names of its contributors + * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * diff --git a/freebsd/sys/dev/usb/wlan/if_rsu.c b/freebsd/sys/dev/usb/wlan/if_rsu.c index 201e75f3..21d67465 100644 --- a/freebsd/sys/dev/usb/wlan/if_rsu.c +++ b/freebsd/sys/dev/usb/wlan/if_rsu.c @@ -528,6 +528,12 @@ rsu_attach(device_t self) sc->sc_ntxstream = 2; rft = "2T2R"; break; + case 0x3: /* "green" NIC */ + sc->sc_rftype = RTL8712_RFCONFIG_1T2R; + sc->sc_nrxstream = 2; + sc->sc_ntxstream = 1; + rft = "1T2R ('green')"; + break; default: device_printf(sc->sc_dev, "%s: unknown board type (rfconfig=0x%02x)\n", diff --git a/freebsd/sys/dev/usb/wlan/if_rum.c b/freebsd/sys/dev/usb/wlan/if_rum.c index 19155ec2..897f9c00 100644 --- a/freebsd/sys/dev/usb/wlan/if_rum.c +++ b/freebsd/sys/dev/usb/wlan/if_rum.c @@ -1505,11 +1505,10 @@ rum_tx_crypto_flags(struct rum_softc *sc, struct ieee80211_node *ni, static int rum_tx_mgt(struct rum_softc *sc, struct mbuf *m0, struct ieee80211_node *ni) { - struct ieee80211vap *vap = ni->ni_vap; + const struct ieee80211_txparam *tp = ni->ni_txparms; struct ieee80211com *ic = &sc->sc_ic; struct rum_tx_data *data; struct ieee80211_frame *wh; - const struct ieee80211_txparam *tp; struct ieee80211_key *k = NULL; uint32_t flags = 0; uint16_t dur; @@ -1539,8 +1538,6 @@ rum_tx_mgt(struct rum_softc *sc, struct mbuf *m0, struct ieee80211_node *ni) wh = mtod(m0, struct ieee80211_frame *); } - tp = &vap->iv_txparms[ieee80211_chan2mode(ic->ic_curchan)]; - if (!IEEE80211_IS_MULTICAST(wh->i_addr1)) { flags |= RT2573_TX_NEED_ACK; @@ -1644,7 +1641,7 @@ rum_tx_data(struct rum_softc *sc, struct mbuf *m0, struct ieee80211_node *ni) struct ieee80211com *ic = &sc->sc_ic; struct rum_tx_data *data; struct ieee80211_frame *wh; - const struct ieee80211_txparam *tp; + const struct ieee80211_txparam *tp = ni->ni_txparms; struct ieee80211_key *k = NULL; uint32_t flags = 0; uint16_t dur; @@ -1663,13 +1660,12 @@ rum_tx_data(struct rum_softc *sc, struct mbuf *m0, struct ieee80211_node *ni) qos = 0; ac = M_WME_GETAC(m0); - tp = &vap->iv_txparms[ieee80211_chan2mode(ni->ni_chan)]; - if (IEEE80211_IS_MULTICAST(wh->i_addr1)) + if (m0->m_flags & M_EAPOL) + rate = tp->mgmtrate; + else if (IEEE80211_IS_MULTICAST(wh->i_addr1)) rate = tp->mcastrate; else if (tp->ucastrate != IEEE80211_FIXED_RATE_NONE) rate = tp->ucastrate; - else if (m0->m_flags & M_EAPOL) - rate = tp->mgmtrate; else { (void) ieee80211_ratectl_rate(ni, NULL, 0); rate = ni->ni_txrate; @@ -2192,12 +2188,11 @@ rum_set_chan(struct rum_softc *sc, struct ieee80211_channel *c) static void rum_set_maxretry(struct rum_softc *sc, struct ieee80211vap *vap) { - const struct ieee80211_txparam *tp; struct ieee80211_node *ni = vap->iv_bss; + const struct ieee80211_txparam *tp = ni->ni_txparms; struct rum_vap *rvp = RUM_VAP(vap); - tp = &vap->iv_txparms[ieee80211_chan2mode(ni->ni_chan)]; - rvp->maxretry = tp->maxretry < 0xf ? tp->maxretry : 0xf; + rvp->maxretry = MIN(tp->maxretry, 0xf); rum_modbits(sc, RT2573_TXRX_CSR4, RT2573_SHORT_RETRY(rvp->maxretry) | RT2573_LONG_RETRY(rvp->maxretry), diff --git a/freebsd/sys/dev/usb/wlan/if_run.c b/freebsd/sys/dev/usb/wlan/if_run.c index 9983fce2..5af8e859 100644 --- a/freebsd/sys/dev/usb/wlan/if_run.c +++ b/freebsd/sys/dev/usb/wlan/if_run.c @@ -3312,8 +3312,7 @@ run_tx(struct run_softc *sc, struct mbuf *m, struct ieee80211_node *ni) struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = ni->ni_vap; struct ieee80211_frame *wh; - struct ieee80211_channel *chan; - const struct ieee80211_txparam *tp; + const struct ieee80211_txparam *tp = ni->ni_txparms; struct run_node *rn = RUN_NODE(ni); struct run_tx_data *data; struct rt2870_txd *txd; @@ -3362,9 +3361,6 @@ run_tx(struct run_softc *sc, struct mbuf *m, struct ieee80211_node *ni) RUN_DPRINTF(sc, RUN_DEBUG_XMIT, "qos %d\tqid %d\ttid %d\tqflags %x\n", qos, qid, tid, qflags); - chan = (ni->ni_chan != IEEE80211_CHAN_ANYC)?ni->ni_chan:ic->ic_curchan; - tp = &vap->iv_txparms[ieee80211_chan2mode(chan)]; - /* pickup a rate index */ if (IEEE80211_IS_MULTICAST(wh->i_addr1) || type != IEEE80211_FC0_TYPE_DATA || m->m_flags & M_EAPOL) { diff --git a/freebsd/sys/dev/usb/wlan/if_ural.c b/freebsd/sys/dev/usb/wlan/if_ural.c index d6a199ff..0e380ec0 100644 --- a/freebsd/sys/dev/usb/wlan/if_ural.c +++ b/freebsd/sys/dev/usb/wlan/if_ural.c @@ -1072,9 +1072,8 @@ ural_tx_bcn(struct ural_softc *sc, struct mbuf *m0, struct ieee80211_node *ni) static int ural_tx_mgt(struct ural_softc *sc, struct mbuf *m0, struct ieee80211_node *ni) { - struct ieee80211vap *vap = ni->ni_vap; + const struct ieee80211_txparam *tp = ni->ni_txparms; struct ieee80211com *ic = ni->ni_ic; - const struct ieee80211_txparam *tp; struct ural_tx_data *data; struct ieee80211_frame *wh; struct ieee80211_key *k; @@ -1087,8 +1086,6 @@ ural_tx_mgt(struct ural_softc *sc, struct mbuf *m0, struct ieee80211_node *ni) STAILQ_REMOVE_HEAD(&sc->tx_free, next); sc->tx_nfree--; - tp = &vap->iv_txparms[ieee80211_chan2mode(ic->ic_curchan)]; - wh = mtod(m0, struct ieee80211_frame *); if (wh->i_fc[1] & IEEE80211_FC1_PROTECTED) { k = ieee80211_crypto_encap(ni, m0); @@ -1241,7 +1238,7 @@ ural_tx_data(struct ural_softc *sc, struct mbuf *m0, struct ieee80211_node *ni) struct ieee80211com *ic = ni->ni_ic; struct ural_tx_data *data; struct ieee80211_frame *wh; - const struct ieee80211_txparam *tp; + const struct ieee80211_txparam *tp = ni->ni_txparms; struct ieee80211_key *k; uint32_t flags = 0; uint16_t dur; @@ -1251,8 +1248,9 @@ ural_tx_data(struct ural_softc *sc, struct mbuf *m0, struct ieee80211_node *ni) wh = mtod(m0, struct ieee80211_frame *); - tp = &vap->iv_txparms[ieee80211_chan2mode(ni->ni_chan)]; - if (IEEE80211_IS_MULTICAST(wh->i_addr1)) + if (m0->m_flags & M_EAPOL) + rate = tp->mgmtrate; + else if (IEEE80211_IS_MULTICAST(wh->i_addr1)) rate = tp->mcastrate; else if (tp->ucastrate != IEEE80211_FIXED_RATE_NONE) rate = tp->ucastrate; diff --git a/freebsd/sys/dev/usb/wlan/if_urtw.c b/freebsd/sys/dev/usb/wlan/if_urtw.c index 501cc50c..74fcee28 100644 --- a/freebsd/sys/dev/usb/wlan/if_urtw.c +++ b/freebsd/sys/dev/usb/wlan/if_urtw.c @@ -1645,7 +1645,7 @@ urtw_tx_start(struct urtw_softc *sc, struct ieee80211_node *ni, struct mbuf *m0, { struct ieee80211_frame *wh = mtod(m0, struct ieee80211_frame *); struct ieee80211_key *k; - const struct ieee80211_txparam *tp; + const struct ieee80211_txparam *tp = ni->ni_txparms; struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = ni->ni_vap; struct usb_xfer *rtl8187b_pipes[URTW_8187B_TXPIPE_MAX] = { @@ -1692,11 +1692,10 @@ urtw_tx_start(struct urtw_softc *sc, struct ieee80211_node *ni, struct mbuf *m0, } if ((wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK) == IEEE80211_FC0_TYPE_MGT || - (wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK) == IEEE80211_FC0_TYPE_CTL) { - tp = &vap->iv_txparms[ieee80211_chan2mode(ic->ic_curchan)]; + (wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK) == IEEE80211_FC0_TYPE_CTL || + (m0->m_flags & M_EAPOL) != 0) { rate = tp->mgmtrate; } else { - tp = &vap->iv_txparms[ieee80211_chan2mode(ni->ni_chan)]; /* for data frames */ if (IEEE80211_IS_MULTICAST(wh->i_addr1)) rate = tp->mcastrate; diff --git a/freebsd/sys/dev/usb/wlan/if_zyd.c b/freebsd/sys/dev/usb/wlan/if_zyd.c index f935bfc9..1208d185 100644 --- a/freebsd/sys/dev/usb/wlan/if_zyd.c +++ b/freebsd/sys/dev/usb/wlan/if_zyd.c @@ -2441,7 +2441,7 @@ zyd_tx_start(struct zyd_softc *sc, struct mbuf *m0, struct ieee80211_node *ni) struct zyd_tx_desc *desc; struct zyd_tx_data *data; struct ieee80211_frame *wh; - const struct ieee80211_txparam *tp; + const struct ieee80211_txparam *tp = ni->ni_txparms; struct ieee80211_key *k; int rate, totlen; static const uint8_t ratediv[] = ZYD_TX_RATEDIV; @@ -2455,11 +2455,10 @@ zyd_tx_start(struct zyd_softc *sc, struct mbuf *m0, struct ieee80211_node *ni) sc->tx_nfree--; if ((wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK) == IEEE80211_FC0_TYPE_MGT || - (wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK) == IEEE80211_FC0_TYPE_CTL) { - tp = &vap->iv_txparms[ieee80211_chan2mode(ic->ic_curchan)]; + (wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK) == IEEE80211_FC0_TYPE_CTL || + (m0->m_flags & M_EAPOL) != 0) { rate = tp->mgmtrate; } else { - tp = &vap->iv_txparms[ieee80211_chan2mode(ni->ni_chan)]; /* for data frames */ if (IEEE80211_IS_MULTICAST(wh->i_addr1)) rate = tp->mcastrate; @@ -2584,10 +2583,10 @@ zyd_start(struct zyd_softc *sc) while (sc->tx_nfree > 0 && (m = mbufq_dequeue(&sc->sc_snd)) != NULL) { ni = (struct ieee80211_node *)m->m_pkthdr.rcvif; if (zyd_tx_start(sc, m, ni) != 0) { - ieee80211_free_node(ni); m_freem(m); if_inc_counter(ni->ni_vap->iv_ifp, IFCOUNTER_OERRORS, 1); + ieee80211_free_node(ni); break; } } |