diff options
| author | Sebastian Huber <sebastian.huber@embedded-brains.de> | 2016-10-07 15:10:20 +0200 |
|---|---|---|
| committer | Sebastian Huber <sebastian.huber@embedded-brains.de> | 2017-01-10 09:53:31 +0100 |
| commit | c40e45b75eb76d79a05c7fa85c1fa9b5c728a12f (patch) | |
| tree | ad4f2519067709f00ab98b3c591186c26dc3a21f /freebsd/sys/dev/e1000 | |
| parent | userspace-header-gen.py: Simplify program ports (diff) | |
| download | rtems-libbsd-c40e45b75eb76d79a05c7fa85c1fa9b5c728a12f.tar.bz2 | |
Update to FreeBSD head 2016-08-23
Git mirror commit 9fe7c416e6abb28b1398fd3e5687099846800cfd.
Diffstat (limited to 'freebsd/sys/dev/e1000')
41 files changed, 3816 insertions, 1745 deletions
diff --git a/freebsd/sys/dev/e1000/e1000_80003es2lan.c b/freebsd/sys/dev/e1000/e1000_80003es2lan.c index 62e9fc42..bd666441 100644 --- a/freebsd/sys/dev/e1000/e1000_80003es2lan.c +++ b/freebsd/sys/dev/e1000/e1000_80003es2lan.c @@ -2,7 +2,7 @@ /****************************************************************************** - Copyright (c) 2001-2013, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -853,11 +853,17 @@ static s32 e1000_reset_hw_80003es2lan(struct e1000_hw *hw) e1000_release_phy_80003es2lan(hw); /* Disable IBIST slave mode (far-end loopback) */ - e1000_read_kmrn_reg_80003es2lan(hw, E1000_KMRNCTRLSTA_INBAND_PARAM, - &kum_reg_data); - kum_reg_data |= E1000_KMRNCTRLSTA_IBIST_DISABLE; - e1000_write_kmrn_reg_80003es2lan(hw, E1000_KMRNCTRLSTA_INBAND_PARAM, - kum_reg_data); + ret_val = e1000_read_kmrn_reg_80003es2lan(hw, + E1000_KMRNCTRLSTA_INBAND_PARAM, &kum_reg_data); + if (!ret_val) { + kum_reg_data |= E1000_KMRNCTRLSTA_IBIST_DISABLE; + ret_val = e1000_write_kmrn_reg_80003es2lan(hw, + E1000_KMRNCTRLSTA_INBAND_PARAM, + kum_reg_data); + if (ret_val) + DEBUGOUT("Error disabling far-end loopback\n"); + } else + DEBUGOUT("Error disabling far-end loopback\n"); ret_val = e1000_get_auto_rd_done_generic(hw); if (ret_val) @@ -913,11 +919,18 @@ static s32 e1000_init_hw_80003es2lan(struct e1000_hw *hw) return ret_val; /* Disable IBIST slave mode (far-end loopback) */ - e1000_read_kmrn_reg_80003es2lan(hw, E1000_KMRNCTRLSTA_INBAND_PARAM, - &kum_reg_data); - kum_reg_data |= E1000_KMRNCTRLSTA_IBIST_DISABLE; - e1000_write_kmrn_reg_80003es2lan(hw, E1000_KMRNCTRLSTA_INBAND_PARAM, - kum_reg_data); + ret_val = + e1000_read_kmrn_reg_80003es2lan(hw, E1000_KMRNCTRLSTA_INBAND_PARAM, + &kum_reg_data); + if (!ret_val) { + kum_reg_data |= E1000_KMRNCTRLSTA_IBIST_DISABLE; + ret_val = e1000_write_kmrn_reg_80003es2lan(hw, + E1000_KMRNCTRLSTA_INBAND_PARAM, + kum_reg_data); + if (ret_val) + DEBUGOUT("Error disabling far-end loopback\n"); + } else + DEBUGOUT("Error disabling far-end loopback\n"); /* Set the transmit descriptor write-back policy */ reg_data = E1000_READ_REG(hw, E1000_TXDCTL(0)); diff --git a/freebsd/sys/dev/e1000/e1000_80003es2lan.h b/freebsd/sys/dev/e1000/e1000_80003es2lan.h index 3807e463..89b15512 100644 --- a/freebsd/sys/dev/e1000/e1000_80003es2lan.h +++ b/freebsd/sys/dev/e1000/e1000_80003es2lan.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2013, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/freebsd/sys/dev/e1000/e1000_82540.c b/freebsd/sys/dev/e1000/e1000_82540.c index c769c901..e6311762 100644 --- a/freebsd/sys/dev/e1000/e1000_82540.c +++ b/freebsd/sys/dev/e1000/e1000_82540.c @@ -2,7 +2,7 @@ /****************************************************************************** - Copyright (c) 2001-2011, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -68,7 +68,7 @@ static s32 e1000_read_mac_addr_82540(struct e1000_hw *hw); static s32 e1000_init_phy_params_82540(struct e1000_hw *hw) { struct e1000_phy_info *phy = &hw->phy; - s32 ret_val = E1000_SUCCESS; + s32 ret_val; phy->addr = 1; phy->autoneg_mask = AUTONEG_ADVERTISE_SPEED_DEFAULT; @@ -331,7 +331,7 @@ static s32 e1000_init_hw_82540(struct e1000_hw *hw) { struct e1000_mac_info *mac = &hw->mac; u32 txdctl, ctrl_ext; - s32 ret_val = E1000_SUCCESS; + s32 ret_val; u16 i; DEBUGFUNC("e1000_init_hw_82540"); @@ -413,7 +413,7 @@ static s32 e1000_init_hw_82540(struct e1000_hw *hw) static s32 e1000_setup_copper_link_82540(struct e1000_hw *hw) { u32 ctrl; - s32 ret_val = E1000_SUCCESS; + s32 ret_val; u16 data; DEBUGFUNC("e1000_setup_copper_link_82540"); @@ -500,7 +500,7 @@ out: **/ static s32 e1000_adjust_serdes_amplitude_82540(struct e1000_hw *hw) { - s32 ret_val = E1000_SUCCESS; + s32 ret_val; u16 nvm_data; DEBUGFUNC("e1000_adjust_serdes_amplitude_82540"); @@ -530,7 +530,7 @@ out: **/ static s32 e1000_set_vco_speed_82540(struct e1000_hw *hw) { - s32 ret_val = E1000_SUCCESS; + s32 ret_val; u16 default_page = 0; u16 phy_data; diff --git a/freebsd/sys/dev/e1000/e1000_82541.c b/freebsd/sys/dev/e1000/e1000_82541.c index 6bfbb33b..770b3b7b 100644 --- a/freebsd/sys/dev/e1000/e1000_82541.c +++ b/freebsd/sys/dev/e1000/e1000_82541.c @@ -2,7 +2,7 @@ /****************************************************************************** - Copyright (c) 2001-2011, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -51,34 +51,34 @@ static s32 e1000_init_mac_params_82541(struct e1000_hw *hw); static s32 e1000_reset_hw_82541(struct e1000_hw *hw); static s32 e1000_init_hw_82541(struct e1000_hw *hw); static s32 e1000_get_link_up_info_82541(struct e1000_hw *hw, u16 *speed, - u16 *duplex); + u16 *duplex); static s32 e1000_phy_hw_reset_82541(struct e1000_hw *hw); static s32 e1000_setup_copper_link_82541(struct e1000_hw *hw); static s32 e1000_check_for_link_82541(struct e1000_hw *hw); static s32 e1000_get_cable_length_igp_82541(struct e1000_hw *hw); static s32 e1000_set_d3_lplu_state_82541(struct e1000_hw *hw, - bool active); + bool active); static s32 e1000_setup_led_82541(struct e1000_hw *hw); static s32 e1000_cleanup_led_82541(struct e1000_hw *hw); static void e1000_clear_hw_cntrs_82541(struct e1000_hw *hw); static s32 e1000_read_mac_addr_82541(struct e1000_hw *hw); static s32 e1000_config_dsp_after_link_change_82541(struct e1000_hw *hw, - bool link_up); + bool link_up); static s32 e1000_phy_init_script_82541(struct e1000_hw *hw); static void e1000_power_down_phy_copper_82541(struct e1000_hw *hw); -static const u16 e1000_igp_cable_length_table[] = - { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 10, 10, 10, 10, 10, 10, 10, 20, 20, 20, 20, 20, 25, 25, 25, - 25, 25, 25, 25, 30, 30, 30, 30, 40, 40, 40, 40, 40, 40, 40, 40, - 40, 50, 50, 50, 50, 50, 50, 50, 60, 60, 60, 60, 60, 60, 60, 60, - 60, 70, 70, 70, 70, 70, 70, 80, 80, 80, 80, 80, 80, 90, 90, 90, - 90, 90, 90, 90, 90, 90, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, - 100, 100, 100, 100, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, - 110, 110, 110, 110, 110, 110, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120}; +static const u16 e1000_igp_cable_length_table[] = { + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 10, 10, 10, 10, 10, + 10, 10, 20, 20, 20, 20, 20, 25, 25, 25, 25, 25, 25, 25, 30, 30, 30, 30, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 50, 50, 50, 50, 50, 50, 50, 60, 60, + 60, 60, 60, 60, 60, 60, 60, 70, 70, 70, 70, 70, 70, 80, 80, 80, 80, 80, + 80, 90, 90, 90, 90, 90, 90, 90, 90, 90, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 110, 110, 110, 110, 110, 110, + 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 120, 120, + 120, 120, 120, 120, 120, 120, 120, 120}; #define IGP01E1000_AGC_LENGTH_TABLE_SIZE \ - (sizeof(e1000_igp_cable_length_table) / \ - sizeof(e1000_igp_cable_length_table[0])) + (sizeof(e1000_igp_cable_length_table) / \ + sizeof(e1000_igp_cable_length_table[0])) /** * e1000_init_phy_params_82541 - Init PHY func ptrs. @@ -87,27 +87,27 @@ static const u16 e1000_igp_cable_length_table[] = static s32 e1000_init_phy_params_82541(struct e1000_hw *hw) { struct e1000_phy_info *phy = &hw->phy; - s32 ret_val = E1000_SUCCESS; + s32 ret_val; DEBUGFUNC("e1000_init_phy_params_82541"); - phy->addr = 1; - phy->autoneg_mask = AUTONEG_ADVERTISE_SPEED_DEFAULT; - phy->reset_delay_us = 10000; - phy->type = e1000_phy_igp; + phy->addr = 1; + phy->autoneg_mask = AUTONEG_ADVERTISE_SPEED_DEFAULT; + phy->reset_delay_us = 10000; + phy->type = e1000_phy_igp; /* Function Pointers */ - phy->ops.check_polarity = e1000_check_polarity_igp; - phy->ops.force_speed_duplex = e1000_phy_force_speed_duplex_igp; - phy->ops.get_cable_length = e1000_get_cable_length_igp_82541; - phy->ops.get_cfg_done = e1000_get_cfg_done_generic; - phy->ops.get_info = e1000_get_phy_info_igp; - phy->ops.read_reg = e1000_read_phy_reg_igp; - phy->ops.reset = e1000_phy_hw_reset_82541; - phy->ops.set_d3_lplu_state = e1000_set_d3_lplu_state_82541; - phy->ops.write_reg = e1000_write_phy_reg_igp; - phy->ops.power_up = e1000_power_up_phy_copper; - phy->ops.power_down = e1000_power_down_phy_copper_82541; + phy->ops.check_polarity = e1000_check_polarity_igp; + phy->ops.force_speed_duplex = e1000_phy_force_speed_duplex_igp; + phy->ops.get_cable_length = e1000_get_cable_length_igp_82541; + phy->ops.get_cfg_done = e1000_get_cfg_done_generic; + phy->ops.get_info = e1000_get_phy_info_igp; + phy->ops.read_reg = e1000_read_phy_reg_igp; + phy->ops.reset = e1000_phy_hw_reset_82541; + phy->ops.set_d3_lplu_state = e1000_set_d3_lplu_state_82541; + phy->ops.write_reg = e1000_write_phy_reg_igp; + phy->ops.power_up = e1000_power_up_phy_copper; + phy->ops.power_down = e1000_power_down_phy_copper_82541; ret_val = e1000_get_phy_id(hw); if (ret_val) @@ -129,8 +129,8 @@ out: **/ static s32 e1000_init_nvm_params_82541(struct e1000_hw *hw) { - struct e1000_nvm_info *nvm = &hw->nvm; - s32 ret_val = E1000_SUCCESS; + struct e1000_nvm_info *nvm = &hw->nvm; + s32 ret_val = E1000_SUCCESS; u32 eecd = E1000_READ_REG(hw, E1000_EECD); u16 size; @@ -154,28 +154,25 @@ static s32 e1000_init_nvm_params_82541(struct e1000_hw *hw) eecd &= ~E1000_EECD_SIZE; break; default: - nvm->type = eecd & E1000_EECD_TYPE - ? e1000_nvm_eeprom_spi - : e1000_nvm_eeprom_microwire; + nvm->type = eecd & E1000_EECD_TYPE ? e1000_nvm_eeprom_spi + : e1000_nvm_eeprom_microwire; break; } if (nvm->type == e1000_nvm_eeprom_spi) { - nvm->address_bits = (eecd & E1000_EECD_ADDR_BITS) - ? 16 : 8; - nvm->delay_usec = 1; - nvm->opcode_bits = 8; - nvm->page_size = (eecd & E1000_EECD_ADDR_BITS) - ? 32 : 8; + nvm->address_bits = (eecd & E1000_EECD_ADDR_BITS) ? 16 : 8; + nvm->delay_usec = 1; + nvm->opcode_bits = 8; + nvm->page_size = (eecd & E1000_EECD_ADDR_BITS) ? 32 : 8; /* Function Pointers */ - nvm->ops.acquire = e1000_acquire_nvm_generic; - nvm->ops.read = e1000_read_nvm_spi; - nvm->ops.release = e1000_release_nvm_generic; - nvm->ops.update = e1000_update_nvm_checksum_generic; + nvm->ops.acquire = e1000_acquire_nvm_generic; + nvm->ops.read = e1000_read_nvm_spi; + nvm->ops.release = e1000_release_nvm_generic; + nvm->ops.update = e1000_update_nvm_checksum_generic; nvm->ops.valid_led_default = e1000_valid_led_default_generic; - nvm->ops.validate = e1000_validate_nvm_checksum_generic; - nvm->ops.write = e1000_write_nvm_spi; + nvm->ops.validate = e1000_validate_nvm_checksum_generic; + nvm->ops.write = e1000_write_nvm_spi; /* * nvm->word_size must be discovered after the pointers @@ -198,21 +195,19 @@ static s32 e1000_init_nvm_params_82541(struct e1000_hw *hw) nvm->word_size = 1 << size; } } else { - nvm->address_bits = (eecd & E1000_EECD_ADDR_BITS) - ? 8 : 6; - nvm->delay_usec = 50; - nvm->opcode_bits = 3; - nvm->word_size = (eecd & E1000_EECD_ADDR_BITS) - ? 256 : 64; + nvm->address_bits = (eecd & E1000_EECD_ADDR_BITS) ? 8 : 6; + nvm->delay_usec = 50; + nvm->opcode_bits = 3; + nvm->word_size = (eecd & E1000_EECD_ADDR_BITS) ? 256 : 64; /* Function Pointers */ - nvm->ops.acquire = e1000_acquire_nvm_generic; - nvm->ops.read = e1000_read_nvm_microwire; - nvm->ops.release = e1000_release_nvm_generic; - nvm->ops.update = e1000_update_nvm_checksum_generic; + nvm->ops.acquire = e1000_acquire_nvm_generic; + nvm->ops.read = e1000_read_nvm_microwire; + nvm->ops.release = e1000_release_nvm_generic; + nvm->ops.update = e1000_update_nvm_checksum_generic; nvm->ops.valid_led_default = e1000_valid_led_default_generic; - nvm->ops.validate = e1000_validate_nvm_checksum_generic; - nvm->ops.write = e1000_write_nvm_microwire; + nvm->ops.validate = e1000_validate_nvm_checksum_generic; + nvm->ops.write = e1000_write_nvm_microwire; } out: @@ -302,7 +297,7 @@ void e1000_init_function_pointers_82541(struct e1000_hw *hw) **/ static s32 e1000_reset_hw_82541(struct e1000_hw *hw) { - u32 ledctl, ctrl, icr, manc; + u32 ledctl, ctrl, manc; DEBUGFUNC("e1000_reset_hw_82541"); @@ -324,6 +319,7 @@ static s32 e1000_reset_hw_82541(struct e1000_hw *hw) /* Must reset the Phy before resetting the MAC */ if ((hw->mac.type == e1000_82541) || (hw->mac.type == e1000_82547)) { E1000_WRITE_REG(hw, E1000_CTRL, (ctrl | E1000_CTRL_PHY_RST)); + E1000_WRITE_FLUSH(hw); msec_delay(5); } @@ -366,7 +362,7 @@ static s32 e1000_reset_hw_82541(struct e1000_hw *hw) E1000_WRITE_REG(hw, E1000_IMC, 0xFFFFFFFF); /* Clear any pending interrupt events. */ - icr = E1000_READ_REG(hw, E1000_ICR); + E1000_READ_REG(hw, E1000_ICR); return E1000_SUCCESS; } @@ -392,11 +388,10 @@ static s32 e1000_init_hw_82541(struct e1000_hw *hw) DEBUGOUT("Error initializing identification LED\n"); /* This is not fatal and we should not stop init due to this */ } - + /* Storing the Speed Power Down value for later use */ - ret_val = hw->phy.ops.read_reg(hw, - IGP01E1000_GMII_FIFO, - &dev_spec->spd_default); + ret_val = hw->phy.ops.read_reg(hw, IGP01E1000_GMII_FIFO, + &dev_spec->spd_default); if (ret_val) goto out; @@ -425,7 +420,7 @@ static s32 e1000_init_hw_82541(struct e1000_hw *hw) txdctl = E1000_READ_REG(hw, E1000_TXDCTL(0)); txdctl = (txdctl & ~E1000_TXDCTL_WTHRESH) | - E1000_TXDCTL_FULL_TX_DESC_WB; + E1000_TXDCTL_FULL_TX_DESC_WB; E1000_WRITE_REG(hw, E1000_TXDCTL(0), txdctl); /* @@ -449,7 +444,7 @@ out: * Retrieve the current speed and duplex configuration. **/ static s32 e1000_get_link_up_info_82541(struct e1000_hw *hw, u16 *speed, - u16 *duplex) + u16 *duplex) { struct e1000_phy_info *phy = &hw->phy; s32 ret_val; @@ -551,6 +546,7 @@ static s32 e1000_setup_copper_link_82541(struct e1000_hw *hw) ctrl &= ~(E1000_CTRL_FRCSPD | E1000_CTRL_FRCDPX); E1000_WRITE_REG(hw, E1000_CTRL, ctrl); + /* Earlier revs of the IGP phy require us to force MDI. */ if (hw->mac.type == e1000_82541 || hw->mac.type == e1000_82547) { dev_spec->dsp_config = e1000_dsp_config_disabled; @@ -653,9 +649,8 @@ static s32 e1000_check_for_link_82541(struct e1000_hw *hw) * different link partner. */ ret_val = e1000_config_fc_after_link_up_generic(hw); - if (ret_val) { + if (ret_val) DEBUGOUT("Error configuring flow control\n"); - } out: return ret_val; @@ -673,7 +668,7 @@ out: * gigabit link is achieved to improve link quality. **/ static s32 e1000_config_dsp_after_link_change_82541(struct e1000_hw *hw, - bool link_up) + bool link_up) { struct e1000_phy_info *phy = &hw->phy; struct e1000_dev_spec_82541 *dev_spec = &hw->dev_spec._82541; @@ -681,11 +676,11 @@ static s32 e1000_config_dsp_after_link_change_82541(struct e1000_hw *hw, u32 idle_errs = 0; u16 phy_data, phy_saved_data, speed, duplex, i; u16 ffe_idle_err_timeout = FFE_IDLE_ERR_COUNT_TIMEOUT_20; - u16 dsp_reg_array[IGP01E1000_PHY_CHANNEL_NUM] = - {IGP01E1000_PHY_AGC_PARAM_A, - IGP01E1000_PHY_AGC_PARAM_B, - IGP01E1000_PHY_AGC_PARAM_C, - IGP01E1000_PHY_AGC_PARAM_D}; + u16 dsp_reg_array[IGP01E1000_PHY_CHANNEL_NUM] = { + IGP01E1000_PHY_AGC_PARAM_A, + IGP01E1000_PHY_AGC_PARAM_B, + IGP01E1000_PHY_AGC_PARAM_C, + IGP01E1000_PHY_AGC_PARAM_D}; DEBUGFUNC("e1000_config_dsp_after_link_change_82541"); @@ -710,16 +705,16 @@ static s32 e1000_config_dsp_after_link_change_82541(struct e1000_hw *hw, for (i = 0; i < IGP01E1000_PHY_CHANNEL_NUM; i++) { ret_val = phy->ops.read_reg(hw, - dsp_reg_array[i], - &phy_data); + dsp_reg_array[i], + &phy_data); if (ret_val) goto out; phy_data &= ~IGP01E1000_PHY_EDAC_MU_INDEX; ret_val = phy->ops.write_reg(hw, - dsp_reg_array[i], - phy_data); + dsp_reg_array[i], + phy_data); if (ret_val) goto out; } @@ -739,9 +734,8 @@ static s32 e1000_config_dsp_after_link_change_82541(struct e1000_hw *hw, for (i = 0; i < ffe_idle_err_timeout; i++) { usec_delay(1000); - ret_val = phy->ops.read_reg(hw, - PHY_1000T_STATUS, - &phy_data); + ret_val = phy->ops.read_reg(hw, PHY_1000T_STATUS, + &phy_data); if (ret_val) goto out; @@ -750,8 +744,8 @@ static s32 e1000_config_dsp_after_link_change_82541(struct e1000_hw *hw, dev_spec->ffe_config = e1000_ffe_config_active; ret_val = phy->ops.write_reg(hw, - IGP01E1000_PHY_DSP_FFE, - IGP01E1000_PHY_DSP_FFE_CM_CP); + IGP01E1000_PHY_DSP_FFE, + IGP01E1000_PHY_DSP_FFE_CM_CP); if (ret_val) goto out; break; @@ -759,7 +753,7 @@ static s32 e1000_config_dsp_after_link_change_82541(struct e1000_hw *hw, if (idle_errs) ffe_idle_err_timeout = - FFE_IDLE_ERR_COUNT_TIMEOUT_100; + FFE_IDLE_ERR_COUNT_TIMEOUT_100; } } else { if (dev_spec->dsp_config == e1000_dsp_config_activated) { @@ -767,9 +761,8 @@ static s32 e1000_config_dsp_after_link_change_82541(struct e1000_hw *hw, * Save off the current value of register 0x2F5B * to be restored at the end of the routines. */ - ret_val = phy->ops.read_reg(hw, - 0x2F5B, - &phy_saved_data); + ret_val = phy->ops.read_reg(hw, 0x2F5B, + &phy_saved_data); if (ret_val) goto out; @@ -780,15 +773,14 @@ static s32 e1000_config_dsp_after_link_change_82541(struct e1000_hw *hw, msec_delay_irq(20); - ret_val = phy->ops.write_reg(hw, - 0x0000, - IGP01E1000_IEEE_FORCE_GIG); + ret_val = phy->ops.write_reg(hw, 0x0000, + IGP01E1000_IEEE_FORCE_GIG); if (ret_val) goto out; for (i = 0; i < IGP01E1000_PHY_CHANNEL_NUM; i++) { ret_val = phy->ops.read_reg(hw, - dsp_reg_array[i], - &phy_data); + dsp_reg_array[i], + &phy_data); if (ret_val) goto out; @@ -796,24 +788,22 @@ static s32 e1000_config_dsp_after_link_change_82541(struct e1000_hw *hw, phy_data |= IGP01E1000_PHY_EDAC_SIGN_EXT_9_BITS; ret_val = phy->ops.write_reg(hw, - dsp_reg_array[i], - phy_data); + dsp_reg_array[i], + phy_data); if (ret_val) goto out; } - ret_val = phy->ops.write_reg(hw, - 0x0000, - IGP01E1000_IEEE_RESTART_AUTONEG); + ret_val = phy->ops.write_reg(hw, 0x0000, + IGP01E1000_IEEE_RESTART_AUTONEG); if (ret_val) goto out; msec_delay_irq(20); /* Now enable the transmitter */ - ret_val = phy->ops.write_reg(hw, - 0x2F5B, - phy_saved_data); + ret_val = phy->ops.write_reg(hw, 0x2F5B, + phy_saved_data); if (ret_val) goto out; @@ -840,21 +830,18 @@ static s32 e1000_config_dsp_after_link_change_82541(struct e1000_hw *hw, msec_delay_irq(20); - ret_val = phy->ops.write_reg(hw, - 0x0000, - IGP01E1000_IEEE_FORCE_GIG); + ret_val = phy->ops.write_reg(hw, 0x0000, + IGP01E1000_IEEE_FORCE_GIG); if (ret_val) goto out; - ret_val = phy->ops.write_reg(hw, - IGP01E1000_PHY_DSP_FFE, - IGP01E1000_PHY_DSP_FFE_DEFAULT); + ret_val = phy->ops.write_reg(hw, IGP01E1000_PHY_DSP_FFE, + IGP01E1000_PHY_DSP_FFE_DEFAULT); if (ret_val) goto out; - ret_val = phy->ops.write_reg(hw, - 0x0000, - IGP01E1000_IEEE_RESTART_AUTONEG); + ret_val = phy->ops.write_reg(hw, 0x0000, + IGP01E1000_IEEE_RESTART_AUTONEG); if (ret_val) goto out; @@ -891,11 +878,10 @@ static s32 e1000_get_cable_length_igp_82541(struct e1000_hw *hw) u16 i, data; u16 cur_agc_value, agc_value = 0; u16 min_agc_value = IGP01E1000_AGC_LENGTH_TABLE_SIZE; - u16 agc_reg_array[IGP01E1000_PHY_CHANNEL_NUM] = - {IGP01E1000_PHY_AGC_A, - IGP01E1000_PHY_AGC_B, - IGP01E1000_PHY_AGC_C, - IGP01E1000_PHY_AGC_D}; + u16 agc_reg_array[IGP01E1000_PHY_CHANNEL_NUM] = {IGP01E1000_PHY_AGC_A, + IGP01E1000_PHY_AGC_B, + IGP01E1000_PHY_AGC_C, + IGP01E1000_PHY_AGC_D}; DEBUGFUNC("e1000_get_cable_length_igp_82541"); @@ -931,12 +917,12 @@ static s32 e1000_get_cable_length_igp_82541(struct e1000_hw *hw) } phy->min_cable_length = (e1000_igp_cable_length_table[agc_value] > - IGP01E1000_AGC_RANGE) - ? (e1000_igp_cable_length_table[agc_value] - - IGP01E1000_AGC_RANGE) - : 0; + IGP01E1000_AGC_RANGE) + ? (e1000_igp_cable_length_table[agc_value] - + IGP01E1000_AGC_RANGE) + : 0; phy->max_cable_length = e1000_igp_cable_length_table[agc_value] + - IGP01E1000_AGC_RANGE; + IGP01E1000_AGC_RANGE; phy->cable_length = (phy->min_cable_length + phy->max_cable_length) / 2; @@ -994,50 +980,48 @@ static s32 e1000_set_d3_lplu_state_82541(struct e1000_hw *hw, bool active) */ if (phy->smart_speed == e1000_smart_speed_on) { ret_val = phy->ops.read_reg(hw, - IGP01E1000_PHY_PORT_CONFIG, - &data); + IGP01E1000_PHY_PORT_CONFIG, + &data); if (ret_val) goto out; data |= IGP01E1000_PSCFR_SMART_SPEED; ret_val = phy->ops.write_reg(hw, - IGP01E1000_PHY_PORT_CONFIG, - data); + IGP01E1000_PHY_PORT_CONFIG, + data); if (ret_val) goto out; } else if (phy->smart_speed == e1000_smart_speed_off) { ret_val = phy->ops.read_reg(hw, - IGP01E1000_PHY_PORT_CONFIG, - &data); + IGP01E1000_PHY_PORT_CONFIG, + &data); if (ret_val) goto out; data &= ~IGP01E1000_PSCFR_SMART_SPEED; ret_val = phy->ops.write_reg(hw, - IGP01E1000_PHY_PORT_CONFIG, - data); + IGP01E1000_PHY_PORT_CONFIG, + data); if (ret_val) goto out; } } else if ((phy->autoneg_advertised == E1000_ALL_SPEED_DUPLEX) || - (phy->autoneg_advertised == E1000_ALL_NOT_GIG) || - (phy->autoneg_advertised == E1000_ALL_10_SPEED)) { + (phy->autoneg_advertised == E1000_ALL_NOT_GIG) || + (phy->autoneg_advertised == E1000_ALL_10_SPEED)) { data |= IGP01E1000_GMII_FLEX_SPD; ret_val = phy->ops.write_reg(hw, IGP01E1000_GMII_FIFO, data); if (ret_val) goto out; /* When LPLU is enabled, we should disable SmartSpeed */ - ret_val = phy->ops.read_reg(hw, - IGP01E1000_PHY_PORT_CONFIG, - &data); + ret_val = phy->ops.read_reg(hw, IGP01E1000_PHY_PORT_CONFIG, + &data); if (ret_val) goto out; data &= ~IGP01E1000_PSCFR_SMART_SPEED; - ret_val = phy->ops.write_reg(hw, - IGP01E1000_PHY_PORT_CONFIG, - data); + ret_val = phy->ops.write_reg(hw, IGP01E1000_PHY_PORT_CONFIG, + data); } out: @@ -1058,16 +1042,14 @@ static s32 e1000_setup_led_82541(struct e1000_hw *hw) DEBUGFUNC("e1000_setup_led_82541"); - ret_val = hw->phy.ops.read_reg(hw, - IGP01E1000_GMII_FIFO, - &dev_spec->spd_default); + ret_val = hw->phy.ops.read_reg(hw, IGP01E1000_GMII_FIFO, + &dev_spec->spd_default); if (ret_val) goto out; - ret_val = hw->phy.ops.write_reg(hw, - IGP01E1000_GMII_FIFO, - (u16)(dev_spec->spd_default & - ~IGP01E1000_GMII_SPD)); + ret_val = hw->phy.ops.write_reg(hw, IGP01E1000_GMII_FIFO, + (u16)(dev_spec->spd_default & + ~IGP01E1000_GMII_SPD)); if (ret_val) goto out; @@ -1091,9 +1073,8 @@ static s32 e1000_cleanup_led_82541(struct e1000_hw *hw) DEBUGFUNC("e1000_cleanup_led_82541"); - ret_val = hw->phy.ops.write_reg(hw, - IGP01E1000_GMII_FIFO, - dev_spec->spd_default); + ret_val = hw->phy.ops.write_reg(hw, IGP01E1000_GMII_FIFO, + dev_spec->spd_default); if (ret_val) goto out; @@ -1180,14 +1161,12 @@ static s32 e1000_phy_init_script_82541(struct e1000_hw *hw) u16 fused, fine, coarse; /* Move to analog registers page */ - hw->phy.ops.read_reg(hw, - IGP01E1000_ANALOG_SPARE_FUSE_STATUS, - &fused); + hw->phy.ops.read_reg(hw, IGP01E1000_ANALOG_SPARE_FUSE_STATUS, + &fused); if (!(fused & IGP01E1000_ANALOG_SPARE_FUSE_ENABLED)) { - hw->phy.ops.read_reg(hw, - IGP01E1000_ANALOG_FUSE_STATUS, - &fused); + hw->phy.ops.read_reg(hw, IGP01E1000_ANALOG_FUSE_STATUS, + &fused); fine = fused & IGP01E1000_ANALOG_FUSE_FINE_MASK; coarse = fused & IGP01E1000_ANALOG_FUSE_COARSE_MASK; @@ -1196,19 +1175,19 @@ static s32 e1000_phy_init_script_82541(struct e1000_hw *hw) coarse -= IGP01E1000_ANALOG_FUSE_COARSE_10; fine -= IGP01E1000_ANALOG_FUSE_FINE_1; } else if (coarse == - IGP01E1000_ANALOG_FUSE_COARSE_THRESH) + IGP01E1000_ANALOG_FUSE_COARSE_THRESH) fine -= IGP01E1000_ANALOG_FUSE_FINE_10; fused = (fused & IGP01E1000_ANALOG_FUSE_POLY_MASK) | - (fine & IGP01E1000_ANALOG_FUSE_FINE_MASK) | - (coarse & IGP01E1000_ANALOG_FUSE_COARSE_MASK); + (fine & IGP01E1000_ANALOG_FUSE_FINE_MASK) | + (coarse & IGP01E1000_ANALOG_FUSE_COARSE_MASK); hw->phy.ops.write_reg(hw, - IGP01E1000_ANALOG_FUSE_CONTROL, - fused); + IGP01E1000_ANALOG_FUSE_CONTROL, + fused); hw->phy.ops.write_reg(hw, - IGP01E1000_ANALOG_FUSE_BYPASS, - IGP01E1000_ANALOG_FUSE_ENABLE_SW_CONTROL); + IGP01E1000_ANALOG_FUSE_BYPASS, + IGP01E1000_ANALOG_FUSE_ENABLE_SW_CONTROL); } } diff --git a/freebsd/sys/dev/e1000/e1000_82541.h b/freebsd/sys/dev/e1000/e1000_82541.h index 3b6b9612..7a01fd43 100644 --- a/freebsd/sys/dev/e1000/e1000_82541.h +++ b/freebsd/sys/dev/e1000/e1000_82541.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2008, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -37,55 +37,55 @@ #define NVM_WORD_SIZE_BASE_SHIFT_82541 (NVM_WORD_SIZE_BASE_SHIFT + 1) -#define IGP01E1000_PHY_CHANNEL_NUM 4 +#define IGP01E1000_PHY_CHANNEL_NUM 4 -#define IGP01E1000_PHY_AGC_A 0x1172 -#define IGP01E1000_PHY_AGC_B 0x1272 -#define IGP01E1000_PHY_AGC_C 0x1472 -#define IGP01E1000_PHY_AGC_D 0x1872 +#define IGP01E1000_PHY_AGC_A 0x1172 +#define IGP01E1000_PHY_AGC_B 0x1272 +#define IGP01E1000_PHY_AGC_C 0x1472 +#define IGP01E1000_PHY_AGC_D 0x1872 -#define IGP01E1000_PHY_AGC_PARAM_A 0x1171 -#define IGP01E1000_PHY_AGC_PARAM_B 0x1271 -#define IGP01E1000_PHY_AGC_PARAM_C 0x1471 -#define IGP01E1000_PHY_AGC_PARAM_D 0x1871 +#define IGP01E1000_PHY_AGC_PARAM_A 0x1171 +#define IGP01E1000_PHY_AGC_PARAM_B 0x1271 +#define IGP01E1000_PHY_AGC_PARAM_C 0x1471 +#define IGP01E1000_PHY_AGC_PARAM_D 0x1871 -#define IGP01E1000_PHY_EDAC_MU_INDEX 0xC000 -#define IGP01E1000_PHY_EDAC_SIGN_EXT_9_BITS 0x8000 +#define IGP01E1000_PHY_EDAC_MU_INDEX 0xC000 +#define IGP01E1000_PHY_EDAC_SIGN_EXT_9_BITS 0x8000 -#define IGP01E1000_PHY_DSP_RESET 0x1F33 +#define IGP01E1000_PHY_DSP_RESET 0x1F33 -#define IGP01E1000_PHY_DSP_FFE 0x1F35 -#define IGP01E1000_PHY_DSP_FFE_CM_CP 0x0069 -#define IGP01E1000_PHY_DSP_FFE_DEFAULT 0x002A +#define IGP01E1000_PHY_DSP_FFE 0x1F35 +#define IGP01E1000_PHY_DSP_FFE_CM_CP 0x0069 +#define IGP01E1000_PHY_DSP_FFE_DEFAULT 0x002A -#define IGP01E1000_IEEE_FORCE_GIG 0x0140 -#define IGP01E1000_IEEE_RESTART_AUTONEG 0x3300 +#define IGP01E1000_IEEE_FORCE_GIG 0x0140 +#define IGP01E1000_IEEE_RESTART_AUTONEG 0x3300 -#define IGP01E1000_AGC_LENGTH_SHIFT 7 -#define IGP01E1000_AGC_RANGE 10 +#define IGP01E1000_AGC_LENGTH_SHIFT 7 +#define IGP01E1000_AGC_RANGE 10 -#define FFE_IDLE_ERR_COUNT_TIMEOUT_20 20 -#define FFE_IDLE_ERR_COUNT_TIMEOUT_100 100 +#define FFE_IDLE_ERR_COUNT_TIMEOUT_20 20 +#define FFE_IDLE_ERR_COUNT_TIMEOUT_100 100 -#define IGP01E1000_ANALOG_FUSE_STATUS 0x20D0 -#define IGP01E1000_ANALOG_SPARE_FUSE_STATUS 0x20D1 -#define IGP01E1000_ANALOG_FUSE_CONTROL 0x20DC -#define IGP01E1000_ANALOG_FUSE_BYPASS 0x20DE +#define IGP01E1000_ANALOG_FUSE_STATUS 0x20D0 +#define IGP01E1000_ANALOG_SPARE_FUSE_STATUS 0x20D1 +#define IGP01E1000_ANALOG_FUSE_CONTROL 0x20DC +#define IGP01E1000_ANALOG_FUSE_BYPASS 0x20DE -#define IGP01E1000_ANALOG_SPARE_FUSE_ENABLED 0x0100 -#define IGP01E1000_ANALOG_FUSE_FINE_MASK 0x0F80 -#define IGP01E1000_ANALOG_FUSE_COARSE_MASK 0x0070 -#define IGP01E1000_ANALOG_FUSE_COARSE_THRESH 0x0040 -#define IGP01E1000_ANALOG_FUSE_COARSE_10 0x0010 -#define IGP01E1000_ANALOG_FUSE_FINE_1 0x0080 -#define IGP01E1000_ANALOG_FUSE_FINE_10 0x0500 -#define IGP01E1000_ANALOG_FUSE_POLY_MASK 0xF000 +#define IGP01E1000_ANALOG_SPARE_FUSE_ENABLED 0x0100 +#define IGP01E1000_ANALOG_FUSE_FINE_MASK 0x0F80 +#define IGP01E1000_ANALOG_FUSE_COARSE_MASK 0x0070 +#define IGP01E1000_ANALOG_FUSE_COARSE_THRESH 0x0040 +#define IGP01E1000_ANALOG_FUSE_COARSE_10 0x0010 +#define IGP01E1000_ANALOG_FUSE_FINE_1 0x0080 +#define IGP01E1000_ANALOG_FUSE_FINE_10 0x0500 +#define IGP01E1000_ANALOG_FUSE_POLY_MASK 0xF000 #define IGP01E1000_ANALOG_FUSE_ENABLE_SW_CONTROL 0x0002 -#define IGP01E1000_MSE_CHANNEL_D 0x000F -#define IGP01E1000_MSE_CHANNEL_C 0x00F0 -#define IGP01E1000_MSE_CHANNEL_B 0x0F00 -#define IGP01E1000_MSE_CHANNEL_A 0xF000 +#define IGP01E1000_MSE_CHANNEL_D 0x000F +#define IGP01E1000_MSE_CHANNEL_C 0x00F0 +#define IGP01E1000_MSE_CHANNEL_B 0x0F00 +#define IGP01E1000_MSE_CHANNEL_A 0xF000 void e1000_init_script_state_82541(struct e1000_hw *hw, bool state); diff --git a/freebsd/sys/dev/e1000/e1000_82542.c b/freebsd/sys/dev/e1000/e1000_82542.c index 95323394..a21fec73 100644 --- a/freebsd/sys/dev/e1000/e1000_82542.c +++ b/freebsd/sys/dev/e1000/e1000_82542.c @@ -2,7 +2,7 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -64,7 +64,7 @@ static s32 e1000_init_phy_params_82542(struct e1000_hw *hw) DEBUGFUNC("e1000_init_phy_params_82542"); - phy->type = e1000_phy_none; + phy->type = e1000_phy_none; return ret_val; } @@ -79,18 +79,18 @@ static s32 e1000_init_nvm_params_82542(struct e1000_hw *hw) DEBUGFUNC("e1000_init_nvm_params_82542"); - nvm->address_bits = 6; - nvm->delay_usec = 50; - nvm->opcode_bits = 3; - nvm->type = e1000_nvm_eeprom_microwire; - nvm->word_size = 64; + nvm->address_bits = 6; + nvm->delay_usec = 50; + nvm->opcode_bits = 3; + nvm->type = e1000_nvm_eeprom_microwire; + nvm->word_size = 64; /* Function Pointers */ - nvm->ops.read = e1000_read_nvm_microwire; - nvm->ops.release = e1000_stop_nvm; - nvm->ops.write = e1000_write_nvm_microwire; - nvm->ops.update = e1000_update_nvm_checksum_generic; - nvm->ops.validate = e1000_validate_nvm_checksum_generic; + nvm->ops.read = e1000_read_nvm_microwire; + nvm->ops.release = e1000_stop_nvm; + nvm->ops.write = e1000_write_nvm_microwire; + nvm->ops.update = e1000_update_nvm_checksum_generic; + nvm->ops.validate = e1000_validate_nvm_checksum_generic; return E1000_SUCCESS; } @@ -126,7 +126,8 @@ static s32 e1000_init_mac_params_82542(struct e1000_hw *hw) /* link setup */ mac->ops.setup_link = e1000_setup_link_82542; /* phy/fiber/serdes setup */ - mac->ops.setup_physical_interface = e1000_setup_fiber_serdes_link_generic; + mac->ops.setup_physical_interface = + e1000_setup_fiber_serdes_link_generic; /* check for link */ mac->ops.check_for_link = e1000_check_for_fiber_link_generic; /* multicast address update */ @@ -145,7 +146,8 @@ static s32 e1000_init_mac_params_82542(struct e1000_hw *hw) /* clear hardware counters */ mac->ops.clear_hw_cntrs = e1000_clear_hw_cntrs_82542; /* link info */ - mac->ops.get_link_up_info = e1000_get_speed_and_duplex_fiber_serdes_generic; + mac->ops.get_link_up_info = + e1000_get_speed_and_duplex_fiber_serdes_generic; return E1000_SUCCESS; } @@ -317,7 +319,7 @@ static s32 e1000_init_hw_82542(struct e1000_hw *hw) static s32 e1000_setup_link_82542(struct e1000_hw *hw) { struct e1000_mac_info *mac = &hw->mac; - s32 ret_val = E1000_SUCCESS; + s32 ret_val; DEBUGFUNC("e1000_setup_link_82542"); @@ -327,7 +329,7 @@ static s32 e1000_setup_link_82542(struct e1000_hw *hw) hw->fc.requested_mode &= ~e1000_fc_tx_pause; - if (mac->report_tx_early == 1) + if (mac->report_tx_early) hw->fc.requested_mode &= ~e1000_fc_rx_pause; /* @@ -337,7 +339,7 @@ static s32 e1000_setup_link_82542(struct e1000_hw *hw) hw->fc.current_mode = hw->fc.requested_mode; DEBUGOUT1("After fix-ups FlowControl is now = %x\n", - hw->fc.current_mode); + hw->fc.current_mode); /* Call the necessary subroutine to configure the link. */ ret_val = mac->ops.setup_physical_interface(hw); @@ -421,9 +423,8 @@ static int e1000_rar_set_82542(struct e1000_hw *hw, u8 *addr, u32 index) * HW expects these in little endian so we reverse the byte order * from network order (big endian) to little endian */ - rar_low = ((u32) addr[0] | - ((u32) addr[1] << 8) | - ((u32) addr[2] << 16) | ((u32) addr[3] << 24)); + rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) | + ((u32) addr[2] << 16) | ((u32) addr[3] << 24)); rar_high = ((u32) addr[4] | ((u32) addr[5] << 8)); @@ -433,6 +434,7 @@ static int e1000_rar_set_82542(struct e1000_hw *hw, u8 *addr, u32 index) E1000_WRITE_REG_ARRAY(hw, E1000_RA, (index << 1), rar_low); E1000_WRITE_REG_ARRAY(hw, E1000_RA, ((index << 1) + 1), rar_high); + return E1000_SUCCESS; } @@ -565,7 +567,7 @@ static void e1000_clear_hw_cntrs_82542(struct e1000_hw *hw) * * Reads the device MAC address from the EEPROM and stores the value. **/ -static s32 e1000_read_mac_addr_82542(struct e1000_hw *hw) +s32 e1000_read_mac_addr_82542(struct e1000_hw *hw) { s32 ret_val = E1000_SUCCESS; u16 offset, nvm_data, i; diff --git a/freebsd/sys/dev/e1000/e1000_82543.c b/freebsd/sys/dev/e1000/e1000_82543.c index 47cf69c4..4930fc95 100644 --- a/freebsd/sys/dev/e1000/e1000_82543.c +++ b/freebsd/sys/dev/e1000/e1000_82543.c @@ -2,7 +2,7 @@ /****************************************************************************** - Copyright (c) 2001-2011, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -49,9 +49,9 @@ static s32 e1000_init_phy_params_82543(struct e1000_hw *hw); static s32 e1000_init_nvm_params_82543(struct e1000_hw *hw); static s32 e1000_init_mac_params_82543(struct e1000_hw *hw); static s32 e1000_read_phy_reg_82543(struct e1000_hw *hw, u32 offset, - u16 *data); + u16 *data); static s32 e1000_write_phy_reg_82543(struct e1000_hw *hw, u32 offset, - u16 data); + u16 data); static s32 e1000_phy_force_speed_duplex_82543(struct e1000_hw *hw); static s32 e1000_phy_hw_reset_82543(struct e1000_hw *hw); static s32 e1000_reset_hw_82543(struct e1000_hw *hw); @@ -64,7 +64,7 @@ static s32 e1000_check_for_fiber_link_82543(struct e1000_hw *hw); static s32 e1000_led_on_82543(struct e1000_hw *hw); static s32 e1000_led_off_82543(struct e1000_hw *hw); static void e1000_write_vfta_82543(struct e1000_hw *hw, u32 offset, - u32 value); + u32 value); static void e1000_clear_hw_cntrs_82543(struct e1000_hw *hw); static s32 e1000_config_mac_to_phy_82543(struct e1000_hw *hw); static bool e1000_init_phy_disabled_82543(struct e1000_hw *hw); @@ -73,7 +73,7 @@ static s32 e1000_polarity_reversal_workaround_82543(struct e1000_hw *hw); static void e1000_raise_mdi_clk_82543(struct e1000_hw *hw, u32 *ctrl); static u16 e1000_shift_in_mdi_bits_82543(struct e1000_hw *hw); static void e1000_shift_out_mdi_bits_82543(struct e1000_hw *hw, u32 data, - u16 count); + u16 count); static bool e1000_tbi_compatibility_enabled_82543(struct e1000_hw *hw); static void e1000_set_tbi_sbp_82543(struct e1000_hw *hw, bool state); static s32 e1000_read_mac_addr_82543(struct e1000_hw *hw); @@ -91,34 +91,34 @@ static s32 e1000_init_phy_params_82543(struct e1000_hw *hw) DEBUGFUNC("e1000_init_phy_params_82543"); if (hw->phy.media_type != e1000_media_type_copper) { - phy->type = e1000_phy_none; + phy->type = e1000_phy_none; goto out; } else { - phy->ops.power_up = e1000_power_up_phy_copper; - phy->ops.power_down = e1000_power_down_phy_copper; + phy->ops.power_up = e1000_power_up_phy_copper; + phy->ops.power_down = e1000_power_down_phy_copper; } - phy->addr = 1; - phy->autoneg_mask = AUTONEG_ADVERTISE_SPEED_DEFAULT; - phy->reset_delay_us = 10000; - phy->type = e1000_phy_m88; + phy->addr = 1; + phy->autoneg_mask = AUTONEG_ADVERTISE_SPEED_DEFAULT; + phy->reset_delay_us = 10000; + phy->type = e1000_phy_m88; /* Function Pointers */ - phy->ops.check_polarity = e1000_check_polarity_m88; - phy->ops.commit = e1000_phy_sw_reset_generic; - phy->ops.force_speed_duplex = e1000_phy_force_speed_duplex_82543; - phy->ops.get_cable_length = e1000_get_cable_length_m88; - phy->ops.get_cfg_done = e1000_get_cfg_done_generic; - phy->ops.read_reg = (hw->mac.type == e1000_82543) - ? e1000_read_phy_reg_82543 - : e1000_read_phy_reg_m88; - phy->ops.reset = (hw->mac.type == e1000_82543) - ? e1000_phy_hw_reset_82543 - : e1000_phy_hw_reset_generic; - phy->ops.write_reg = (hw->mac.type == e1000_82543) - ? e1000_write_phy_reg_82543 - : e1000_write_phy_reg_m88; - phy->ops.get_info = e1000_get_phy_info_m88; + phy->ops.check_polarity = e1000_check_polarity_m88; + phy->ops.commit = e1000_phy_sw_reset_generic; + phy->ops.force_speed_duplex = e1000_phy_force_speed_duplex_82543; + phy->ops.get_cable_length = e1000_get_cable_length_m88; + phy->ops.get_cfg_done = e1000_get_cfg_done_generic; + phy->ops.read_reg = (hw->mac.type == e1000_82543) + ? e1000_read_phy_reg_82543 + : e1000_read_phy_reg_m88; + phy->ops.reset = (hw->mac.type == e1000_82543) + ? e1000_phy_hw_reset_82543 + : e1000_phy_hw_reset_generic; + phy->ops.write_reg = (hw->mac.type == e1000_82543) + ? e1000_write_phy_reg_82543 + : e1000_write_phy_reg_m88; + phy->ops.get_info = e1000_get_phy_info_m88; /* * The external PHY of the 82543 can be in a funky state. @@ -172,18 +172,18 @@ static s32 e1000_init_nvm_params_82543(struct e1000_hw *hw) DEBUGFUNC("e1000_init_nvm_params_82543"); - nvm->type = e1000_nvm_eeprom_microwire; - nvm->word_size = 64; - nvm->delay_usec = 50; - nvm->address_bits = 6; - nvm->opcode_bits = 3; + nvm->type = e1000_nvm_eeprom_microwire; + nvm->word_size = 64; + nvm->delay_usec = 50; + nvm->address_bits = 6; + nvm->opcode_bits = 3; /* Function Pointers */ - nvm->ops.read = e1000_read_nvm_microwire; - nvm->ops.update = e1000_update_nvm_checksum_generic; + nvm->ops.read = e1000_read_nvm_microwire; + nvm->ops.update = e1000_update_nvm_checksum_generic; nvm->ops.valid_led_default = e1000_valid_led_default_generic; - nvm->ops.validate = e1000_validate_nvm_checksum_generic; - nvm->ops.write = e1000_write_nvm_microwire; + nvm->ops.validate = e1000_validate_nvm_checksum_generic; + nvm->ops.write = e1000_write_nvm_microwire; return E1000_SUCCESS; } @@ -228,19 +228,18 @@ static s32 e1000_init_mac_params_82543(struct e1000_hw *hw) mac->ops.setup_link = e1000_setup_link_82543; /* physical interface setup */ mac->ops.setup_physical_interface = - (hw->phy.media_type == e1000_media_type_copper) - ? e1000_setup_copper_link_82543 - : e1000_setup_fiber_link_82543; + (hw->phy.media_type == e1000_media_type_copper) + ? e1000_setup_copper_link_82543 : e1000_setup_fiber_link_82543; /* check for link */ mac->ops.check_for_link = - (hw->phy.media_type == e1000_media_type_copper) - ? e1000_check_for_copper_link_82543 - : e1000_check_for_fiber_link_82543; + (hw->phy.media_type == e1000_media_type_copper) + ? e1000_check_for_copper_link_82543 + : e1000_check_for_fiber_link_82543; /* link info */ mac->ops.get_link_up_info = - (hw->phy.media_type == e1000_media_type_copper) - ? e1000_get_speed_and_duplex_copper_generic - : e1000_get_speed_and_duplex_fiber_serdes_generic; + (hw->phy.media_type == e1000_media_type_copper) + ? e1000_get_speed_and_duplex_copper_generic + : e1000_get_speed_and_duplex_fiber_serdes_generic; /* multicast address update */ mac->ops.update_mc_addr_list = e1000_update_mc_addr_list_generic; /* writing VFTA */ @@ -297,8 +296,7 @@ static bool e1000_tbi_compatibility_enabled_82543(struct e1000_hw *hw) goto out; } - state = (dev_spec->tbi_compatibility & TBI_COMPAT_ENABLED) - ? TRUE : FALSE; + state = !!(dev_spec->tbi_compatibility & TBI_COMPAT_ENABLED); out: return state; @@ -350,8 +348,7 @@ bool e1000_tbi_sbp_enabled_82543(struct e1000_hw *hw) goto out; } - state = (dev_spec->tbi_compatibility & TBI_SBP_ENABLED) - ? TRUE : FALSE; + state = !!(dev_spec->tbi_compatibility & TBI_SBP_ENABLED); out: return state; @@ -414,8 +411,8 @@ out: * Adjusts the statistic counters when a frame is accepted by TBI_ACCEPT **/ void e1000_tbi_adjust_stats_82543(struct e1000_hw *hw, - struct e1000_hw_stats *stats, u32 frame_len, - u8 *mac_addr, u32 max_frame_size) + struct e1000_hw_stats *stats, u32 frame_len, + u8 *mac_addr, u32 max_frame_size) { if (!(e1000_tbi_sbp_enabled_82543(hw))) goto out; @@ -427,12 +424,12 @@ void e1000_tbi_adjust_stats_82543(struct e1000_hw *hw, * counters overcount this packet as a CRC error and undercount * the packet as a good packet */ - /* This packet should not be counted as a CRC error. */ + /* This packet should not be counted as a CRC error. */ stats->crcerrs--; - /* This packet does count as a Good Packet Received. */ + /* This packet does count as a Good Packet Received. */ stats->gprc++; - /* Adjust the Good Octets received counters */ + /* Adjust the Good Octets received counters */ stats->gorc += frame_len; /* @@ -448,7 +445,7 @@ void e1000_tbi_adjust_stats_82543(struct e1000_hw *hw, stats->mprc++; /* - * In this case, the hardware has overcounted the number of + * In this case, the hardware has over counted the number of * oversize frames. */ if ((frame_len == max_frame_size) && (stats->roc > 0)) @@ -515,7 +512,7 @@ static s32 e1000_read_phy_reg_82543(struct e1000_hw *hw, u32 offset, u16 *data) * e1000_shift_out_mdi_bits routine five different times. The format * of an MII read instruction consists of a shift out of 14 bits and * is defined as follows: - * <Preamble><SOF><Op Code><Phy Addr><Offset> + * <Preamble><SOF><Op Code><Phy Addr><Offset> * followed by a shift in of 18 bits. This first two bits shifted in * are TurnAround bits used to avoid contention on the MDIO pin when a * READ operation is performed. These two bits are thrown away @@ -574,9 +571,9 @@ static s32 e1000_write_phy_reg_82543(struct e1000_hw *hw, u32 offset, u16 data) * <Preamble><SOF><Op Code><Phy Addr><Reg Addr><Turnaround><Data>. */ mdic = ((PHY_TURNAROUND) | (offset << 2) | (hw->phy.addr << 7) | - (PHY_OP_WRITE << 12) | (PHY_SOF << 14)); + (PHY_OP_WRITE << 12) | (PHY_SOF << 14)); mdic <<= 16; - mdic |= (u32) data; + mdic |= (u32)data; e1000_shift_out_mdi_bits_82543(hw, mdic, 32); @@ -633,7 +630,7 @@ static void e1000_lower_mdi_clk_82543(struct e1000_hw *hw, u32 *ctrl) * In order to do this, "data" must be broken down into bits. **/ static void e1000_shift_out_mdi_bits_82543(struct e1000_hw *hw, u32 data, - u16 count) + u16 count) { u32 ctrl, mask; @@ -644,7 +641,7 @@ static void e1000_shift_out_mdi_bits_82543(struct e1000_hw *hw, u32 data, * into bits. */ mask = 0x01; - mask <<= (count -1); + mask <<= (count - 1); ctrl = E1000_READ_REG(hw, E1000_CTRL); @@ -658,8 +655,10 @@ static void e1000_shift_out_mdi_bits_82543(struct e1000_hw *hw, u32 data, * A "0" is shifted out to the PHY by setting the MDIO bit to * "0" and then raising and lowering the clock. */ - if (data & mask) ctrl |= E1000_CTRL_MDIO; - else ctrl &= ~E1000_CTRL_MDIO; + if (data & mask) + ctrl |= E1000_CTRL_MDIO; + else + ctrl &= ~E1000_CTRL_MDIO; E1000_WRITE_REG(hw, E1000_CTRL, ctrl); E1000_WRITE_FLUSH(hw); @@ -751,8 +750,8 @@ static s32 e1000_phy_force_speed_duplex_82543(struct e1000_hw *hw) if (ret_val) goto out; - if (!hw->mac.autoneg && - (hw->mac.forced_speed_duplex & E1000_ALL_10_SPEED)) + if (!hw->mac.autoneg && (hw->mac.forced_speed_duplex & + E1000_ALL_10_SPEED)) ret_val = e1000_polarity_reversal_workaround_82543(hw); out: @@ -810,7 +809,7 @@ static s32 e1000_polarity_reversal_workaround_82543(struct e1000_hw *hw) if (ret_val) goto out; - if ((mii_status_reg & ~MII_SR_LINK_STATUS) == 0) + if (!(mii_status_reg & ~MII_SR_LINK_STATUS)) break; msec_delay_irq(100); } @@ -903,7 +902,7 @@ static s32 e1000_phy_hw_reset_82543(struct e1000_hw *hw) **/ static s32 e1000_reset_hw_82543(struct e1000_hw *hw) { - u32 ctrl, icr; + u32 ctrl; s32 ret_val = E1000_SUCCESS; DEBUGFUNC("e1000_reset_hw_82543"); @@ -945,7 +944,7 @@ static s32 e1000_reset_hw_82543(struct e1000_hw *hw) /* Masking off and clearing any pending interrupts */ E1000_WRITE_REG(hw, E1000_IMC, 0xffffffff); - icr = E1000_READ_REG(hw, E1000_ICR); + E1000_READ_REG(hw, E1000_ICR); return ret_val; } @@ -1042,7 +1041,7 @@ static s32 e1000_setup_link_82543(struct e1000_hw *hw) goto out; } ctrl_ext = ((data & NVM_WORD0F_SWPDIO_EXT_MASK) << - NVM_SWDPIO_EXT_SHIFT); + NVM_SWDPIO_EXT_SHIFT); E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); } @@ -1116,10 +1115,8 @@ static s32 e1000_setup_copper_link_82543(struct e1000_hw *hw) * Check link status. Wait up to 100 microseconds for link to become * valid. */ - ret_val = e1000_phy_has_link_generic(hw, - COPPER_LINK_UP_LIMIT, - 10, - &link); + ret_val = e1000_phy_has_link_generic(hw, COPPER_LINK_UP_LIMIT, 10, + &link); if (ret_val) goto out; @@ -1179,11 +1176,10 @@ static s32 e1000_setup_fiber_link_82543(struct e1000_hw *hw) * optics detect a signal. If we have a signal, then poll for a * "Link-Up" indication. */ - if (!(E1000_READ_REG(hw, E1000_CTRL) & E1000_CTRL_SWDPIN1)) { + if (!(E1000_READ_REG(hw, E1000_CTRL) & E1000_CTRL_SWDPIN1)) ret_val = e1000_poll_fiber_serdes_link_generic(hw); - } else { + else DEBUGOUT("No signal detected\n"); - } out: return ret_val; @@ -1277,9 +1273,8 @@ static s32 e1000_check_for_copper_link_82543(struct e1000_hw *hw) * different link partner. */ ret_val = e1000_config_fc_after_link_up_generic(hw); - if (ret_val) { + if (ret_val) DEBUGOUT("Error configuring flow control\n"); - } /* * At this point we know that we are on copper and we have @@ -1361,8 +1356,8 @@ static s32 e1000_check_for_fiber_link_82543(struct e1000_hw *hw) if ((!(ctrl & E1000_CTRL_SWDPIN1)) && (!(status & E1000_STATUS_LU)) && (!(rxcw & E1000_RXCW_C))) { - if (mac->autoneg_failed == 0) { - mac->autoneg_failed = 1; + if (!mac->autoneg_failed) { + mac->autoneg_failed = TRUE; ret_val = 0; goto out; } diff --git a/freebsd/sys/dev/e1000/e1000_82543.h b/freebsd/sys/dev/e1000/e1000_82543.h index 60e5c15d..e8edda5d 100644 --- a/freebsd/sys/dev/e1000/e1000_82543.h +++ b/freebsd/sys/dev/e1000/e1000_82543.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2008, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -35,23 +35,23 @@ #ifndef _E1000_82543_H_ #define _E1000_82543_H_ -#define PHY_PREAMBLE 0xFFFFFFFF -#define PHY_PREAMBLE_SIZE 32 -#define PHY_SOF 0x1 -#define PHY_OP_READ 0x2 -#define PHY_OP_WRITE 0x1 -#define PHY_TURNAROUND 0x2 +#define PHY_PREAMBLE 0xFFFFFFFF +#define PHY_PREAMBLE_SIZE 32 +#define PHY_SOF 0x1 +#define PHY_OP_READ 0x2 +#define PHY_OP_WRITE 0x1 +#define PHY_TURNAROUND 0x2 -#define TBI_COMPAT_ENABLED 0x1 /* Global "knob" for the workaround */ +#define TBI_COMPAT_ENABLED 0x1 /* Global "knob" for the workaround */ /* If TBI_COMPAT_ENABLED, then this is the current state (on/off) */ -#define TBI_SBP_ENABLED 0x2 - +#define TBI_SBP_ENABLED 0x2 + void e1000_tbi_adjust_stats_82543(struct e1000_hw *hw, - struct e1000_hw_stats *stats, - u32 frame_len, u8 *mac_addr, - u32 max_frame_size); + struct e1000_hw_stats *stats, + u32 frame_len, u8 *mac_addr, + u32 max_frame_size); void e1000_set_tbi_compatibility_82543(struct e1000_hw *hw, - bool state); + bool state); bool e1000_tbi_sbp_enabled_82543(struct e1000_hw *hw); #endif diff --git a/freebsd/sys/dev/e1000/e1000_82571.c b/freebsd/sys/dev/e1000/e1000_82571.c index dadd3a82..2b72489a 100644 --- a/freebsd/sys/dev/e1000/e1000_82571.c +++ b/freebsd/sys/dev/e1000/e1000_82571.c @@ -2,7 +2,7 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -398,7 +398,7 @@ static s32 e1000_init_mac_params_82571(struct e1000_hw *hw) } /* Ensure that the inter-port SWSM.SMBI lock bit is clear before - * first NVM or PHY acess. This should be done for single-port + * first NVM or PHY access. This should be done for single-port * devices, and for one port only on dual-port devices so that * for those devices we can still use the SMBI lock to synchronize * inter-port accesses to the PHY & NVM. diff --git a/freebsd/sys/dev/e1000/e1000_82571.h b/freebsd/sys/dev/e1000/e1000_82571.h index 41d5df0e..8e5ca56a 100644 --- a/freebsd/sys/dev/e1000/e1000_82571.h +++ b/freebsd/sys/dev/e1000/e1000_82571.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/freebsd/sys/dev/e1000/e1000_82575.c b/freebsd/sys/dev/e1000/e1000_82575.c index 0e06471f..83116e8e 100644 --- a/freebsd/sys/dev/e1000/e1000_82575.c +++ b/freebsd/sys/dev/e1000/e1000_82575.c @@ -2,7 +2,7 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -280,6 +280,11 @@ static s32 e1000_init_phy_params_82575(struct e1000_hw *hw) if (ret_val) goto out; } + if (phy->id == M88E1543_E_PHY_ID) { + ret_val = e1000_initialize_M88E1543_phy(hw); + if (ret_val) + goto out; + } break; case IGP03E1000_E_PHY_ID: case IGP04E1000_E_PHY_ID: @@ -1053,7 +1058,7 @@ static s32 e1000_acquire_swfw_sync_82575(struct e1000_hw *hw, u16 mask) u32 swmask = mask; u32 fwmask = mask << 16; s32 ret_val = E1000_SUCCESS; - s32 i = 0, timeout = 200; /* FIXME: find real value to use here */ + s32 i = 0, timeout = 200; DEBUGFUNC("e1000_acquire_swfw_sync_82575"); @@ -1237,7 +1242,7 @@ static s32 e1000_check_for_link_media_swap(struct e1000_hw *hw) DEBUGFUNC("e1000_check_for_link_media_swap"); - /* Check the copper medium. */ + /* Check for copper. */ ret_val = phy->ops.write_reg(hw, E1000_M88E1112_PAGE_ADDR, 0); if (ret_val) return ret_val; @@ -1249,7 +1254,7 @@ static s32 e1000_check_for_link_media_swap(struct e1000_hw *hw) if (data & E1000_M88E1112_STATUS_LINK) port = E1000_MEDIA_PORT_COPPER; - /* Check the other medium. */ + /* Check for other. */ ret_val = phy->ops.write_reg(hw, E1000_M88E1112_PAGE_ADDR, 1); if (ret_val) return ret_val; @@ -1258,11 +1263,6 @@ static s32 e1000_check_for_link_media_swap(struct e1000_hw *hw) if (ret_val) return ret_val; - /* reset page to 0 */ - ret_val = phy->ops.write_reg(hw, E1000_M88E1112_PAGE_ADDR, 0); - if (ret_val) - return ret_val; - if (data & E1000_M88E1112_STATUS_LINK) port = E1000_MEDIA_PORT_OTHER; @@ -1270,8 +1270,20 @@ static s32 e1000_check_for_link_media_swap(struct e1000_hw *hw) if (port && (hw->dev_spec._82575.media_port != port)) { hw->dev_spec._82575.media_port = port; hw->dev_spec._82575.media_changed = TRUE; + } + + if (port == E1000_MEDIA_PORT_COPPER) { + /* reset page to 0 */ + ret_val = phy->ops.write_reg(hw, E1000_M88E1112_PAGE_ADDR, 0); + if (ret_val) + return ret_val; + e1000_check_for_link_82575(hw); } else { - ret_val = e1000_check_for_link_82575(hw); + e1000_check_for_link_82575(hw); + /* reset page to 0 */ + ret_val = phy->ops.write_reg(hw, E1000_M88E1112_PAGE_ADDR, 0); + if (ret_val) + return ret_val; } return E1000_SUCCESS; @@ -2128,7 +2140,7 @@ static void e1000_clear_hw_cntrs_82575(struct e1000_hw *hw) * e1000_rx_fifo_flush_82575 - Clean rx fifo after Rx enable * @hw: pointer to the HW structure * - * After rx enable if managability is enabled then there is likely some + * After Rx enable, if manageability is enabled then there is likely some * bad data at the start of the fifo and possibly in the DMA fifo. This * function clears the fifos and flushes any packets that came in as rx was * being enabled. @@ -2138,7 +2150,13 @@ void e1000_rx_fifo_flush_82575(struct e1000_hw *hw) u32 rctl, rlpml, rxdctl[4], rfctl, temp_rctl, rx_enabled; int i, ms_wait; - DEBUGFUNC("e1000_rx_fifo_workaround_82575"); + DEBUGFUNC("e1000_rx_fifo_flush_82575"); + + /* disable IPv6 options as per hardware errata */ + rfctl = E1000_READ_REG(hw, E1000_RFCTL); + rfctl |= E1000_RFCTL_IPV6_EX_DIS; + E1000_WRITE_REG(hw, E1000_RFCTL, rfctl); + if (hw->mac.type != e1000_82575 || !(E1000_READ_REG(hw, E1000_MANC) & E1000_MANC_RCV_TCO_EN)) return; @@ -2166,7 +2184,6 @@ void e1000_rx_fifo_flush_82575(struct e1000_hw *hw) * incoming packets are rejected. Set enable and wait 2ms so that * any packet that was coming in as RCTL.EN was set is flushed */ - rfctl = E1000_READ_REG(hw, E1000_RFCTL); E1000_WRITE_REG(hw, E1000_RFCTL, rfctl & ~E1000_RFCTL_LEF); rlpml = E1000_READ_REG(hw, E1000_RLPML); @@ -2401,7 +2418,7 @@ out: * e1000_reset_mdicnfg_82580 - Reset MDICNFG destination and com_mdio bits * @hw: pointer to the HW structure * - * This resets the the MDICNFG.Destination and MDICNFG.Com_MDIO bits based on + * This resets the MDICNFG.Destination and MDICNFG.Com_MDIO bits based on * the values found in the EEPROM. This addresses an issue in which these * bits are not restored from EEPROM after reset. **/ @@ -2808,7 +2825,7 @@ s32 e1000_read_emi_reg(struct e1000_hw *hw, u16 addr, u16 *data) * e1000_initialize_M88E1512_phy - Initialize M88E1512 PHY * @hw: pointer to the HW structure * - * Initialize Marverl 1512 to work correctly with Avoton. + * Initialize Marvell 1512 to work correctly with Avoton. **/ s32 e1000_initialize_M88E1512_phy(struct e1000_hw *hw) { @@ -2894,13 +2911,114 @@ out: } /** + * e1000_initialize_M88E1543_phy - Initialize M88E1543 PHY + * @hw: pointer to the HW structure + * + * Initialize Marvell 1543 to work correctly with Avoton. + **/ +s32 e1000_initialize_M88E1543_phy(struct e1000_hw *hw) +{ + struct e1000_phy_info *phy = &hw->phy; + s32 ret_val = E1000_SUCCESS; + + DEBUGFUNC("e1000_initialize_M88E1543_phy"); + + /* Check if this is correct PHY. */ + if (phy->id != M88E1543_E_PHY_ID) + goto out; + + /* Switch to PHY page 0xFF. */ + ret_val = phy->ops.write_reg(hw, E1000_M88E1543_PAGE_ADDR, 0x00FF); + if (ret_val) + goto out; + + ret_val = phy->ops.write_reg(hw, E1000_M88E1512_CFG_REG_2, 0x214B); + if (ret_val) + goto out; + + ret_val = phy->ops.write_reg(hw, E1000_M88E1512_CFG_REG_1, 0x2144); + if (ret_val) + goto out; + + ret_val = phy->ops.write_reg(hw, E1000_M88E1512_CFG_REG_2, 0x0C28); + if (ret_val) + goto out; + + ret_val = phy->ops.write_reg(hw, E1000_M88E1512_CFG_REG_1, 0x2146); + if (ret_val) + goto out; + + ret_val = phy->ops.write_reg(hw, E1000_M88E1512_CFG_REG_2, 0xB233); + if (ret_val) + goto out; + + ret_val = phy->ops.write_reg(hw, E1000_M88E1512_CFG_REG_1, 0x214D); + if (ret_val) + goto out; + + ret_val = phy->ops.write_reg(hw, E1000_M88E1512_CFG_REG_2, 0xDC0C); + if (ret_val) + goto out; + + ret_val = phy->ops.write_reg(hw, E1000_M88E1512_CFG_REG_1, 0x2159); + if (ret_val) + goto out; + + /* Switch to PHY page 0xFB. */ + ret_val = phy->ops.write_reg(hw, E1000_M88E1543_PAGE_ADDR, 0x00FB); + if (ret_val) + goto out; + + ret_val = phy->ops.write_reg(hw, E1000_M88E1512_CFG_REG_3, 0xC00D); + if (ret_val) + goto out; + + /* Switch to PHY page 0x12. */ + ret_val = phy->ops.write_reg(hw, E1000_M88E1543_PAGE_ADDR, 0x12); + if (ret_val) + goto out; + + /* Change mode to SGMII-to-Copper */ + ret_val = phy->ops.write_reg(hw, E1000_M88E1512_MODE, 0x8001); + if (ret_val) + goto out; + + /* Switch to PHY page 1. */ + ret_val = phy->ops.write_reg(hw, E1000_M88E1543_PAGE_ADDR, 0x1); + if (ret_val) + goto out; + + /* Change mode to 1000BASE-X/SGMII and autoneg enable; reset */ + ret_val = phy->ops.write_reg(hw, E1000_M88E1543_FIBER_CTRL, 0x9140); + if (ret_val) + goto out; + + /* Return the PHY to page 0. */ + ret_val = phy->ops.write_reg(hw, E1000_M88E1543_PAGE_ADDR, 0); + if (ret_val) + goto out; + + ret_val = phy->ops.commit(hw); + if (ret_val) { + DEBUGOUT("Error committing the PHY changes\n"); + return ret_val; + } + + msec_delay(1000); +out: + return ret_val; +} + +/** * e1000_set_eee_i350 - Enable/disable EEE support * @hw: pointer to the HW structure + * @adv1g: boolean flag enabling 1G EEE advertisement + * @adv100m: boolean flag enabling 100M EEE advertisement * * Enable/disable EEE based on setting in dev_spec structure. * **/ -s32 e1000_set_eee_i350(struct e1000_hw *hw) +s32 e1000_set_eee_i350(struct e1000_hw *hw, bool adv1G, bool adv100M) { u32 ipcnfg, eeer; @@ -2916,7 +3034,16 @@ s32 e1000_set_eee_i350(struct e1000_hw *hw) if (!(hw->dev_spec._82575.eee_disable)) { u32 eee_su = E1000_READ_REG(hw, E1000_EEE_SU); - ipcnfg |= (E1000_IPCNFG_EEE_1G_AN | E1000_IPCNFG_EEE_100M_AN); + if (adv100M) + ipcnfg |= E1000_IPCNFG_EEE_100M_AN; + else + ipcnfg &= ~E1000_IPCNFG_EEE_100M_AN; + + if (adv1G) + ipcnfg |= E1000_IPCNFG_EEE_1G_AN; + else + ipcnfg &= ~E1000_IPCNFG_EEE_1G_AN; + eeer |= (E1000_EEER_TX_LPI_EN | E1000_EEER_RX_LPI_EN | E1000_EEER_LPI_FC); @@ -2940,11 +3067,13 @@ out: /** * e1000_set_eee_i354 - Enable/disable EEE support * @hw: pointer to the HW structure + * @adv1g: boolean flag enabling 1G EEE advertisement + * @adv100m: boolean flag enabling 100M EEE advertisement * * Enable/disable EEE legacy mode based on setting in dev_spec structure. * **/ -s32 e1000_set_eee_i354(struct e1000_hw *hw) +s32 e1000_set_eee_i354(struct e1000_hw *hw, bool adv1G, bool adv100M) { struct e1000_phy_info *phy = &hw->phy; s32 ret_val = E1000_SUCCESS; @@ -2986,8 +3115,16 @@ s32 e1000_set_eee_i354(struct e1000_hw *hw) if (ret_val) goto out; - phy_data |= E1000_EEE_ADV_100_SUPPORTED | - E1000_EEE_ADV_1000_SUPPORTED; + if (adv100M) + phy_data |= E1000_EEE_ADV_100_SUPPORTED; + else + phy_data &= ~E1000_EEE_ADV_100_SUPPORTED; + + if (adv1G) + phy_data |= E1000_EEE_ADV_1000_SUPPORTED; + else + phy_data &= ~E1000_EEE_ADV_1000_SUPPORTED; + ret_val = e1000_write_xmdio_reg(hw, E1000_EEE_ADV_ADDR_I354, E1000_EEE_ADV_DEV_I354, phy_data); diff --git a/freebsd/sys/dev/e1000/e1000_82575.h b/freebsd/sys/dev/e1000/e1000_82575.h index 6569b988..45fe132e 100644 --- a/freebsd/sys/dev/e1000/e1000_82575.h +++ b/freebsd/sys/dev/e1000/e1000_82575.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -495,10 +495,11 @@ void e1000_rlpml_set_vf(struct e1000_hw *, u16); s32 e1000_promisc_set_vf(struct e1000_hw *, enum e1000_promisc_type type); u16 e1000_rxpbs_adjust_82580(u32 data); s32 e1000_read_emi_reg(struct e1000_hw *hw, u16 addr, u16 *data); -s32 e1000_set_eee_i350(struct e1000_hw *); -s32 e1000_set_eee_i354(struct e1000_hw *); +s32 e1000_set_eee_i350(struct e1000_hw *hw, bool adv1G, bool adv100M); +s32 e1000_set_eee_i354(struct e1000_hw *hw, bool adv1G, bool adv100M); s32 e1000_get_eee_status_i354(struct e1000_hw *, bool *); s32 e1000_initialize_M88E1512_phy(struct e1000_hw *hw); +s32 e1000_initialize_M88E1543_phy(struct e1000_hw *hw); /* I2C SDA and SCL timing parameters for standard mode */ #define E1000_I2C_T_HD_STA 4 diff --git a/freebsd/sys/dev/e1000/e1000_api.c b/freebsd/sys/dev/e1000/e1000_api.c index 71315bde..829e7be0 100644 --- a/freebsd/sys/dev/e1000/e1000_api.c +++ b/freebsd/sys/dev/e1000/e1000_api.c @@ -2,7 +2,7 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -301,6 +301,17 @@ s32 e1000_set_mac_type(struct e1000_hw *hw) case E1000_DEV_ID_PCH_I218_V3: mac->type = e1000_pch_lpt; break; + case E1000_DEV_ID_PCH_SPT_I219_LM: + case E1000_DEV_ID_PCH_SPT_I219_V: + case E1000_DEV_ID_PCH_SPT_I219_LM2: + case E1000_DEV_ID_PCH_SPT_I219_V2: + case E1000_DEV_ID_PCH_LBG_I219_LM3: + case E1000_DEV_ID_PCH_SPT_I219_LM4: + case E1000_DEV_ID_PCH_SPT_I219_V4: + case E1000_DEV_ID_PCH_SPT_I219_LM5: + case E1000_DEV_ID_PCH_SPT_I219_V5: + mac->type = e1000_pch_spt; + break; case E1000_DEV_ID_82575EB_COPPER: case E1000_DEV_ID_82575EB_FIBER_SERDES: case E1000_DEV_ID_82575GB_QUAD_COPPER: @@ -451,6 +462,7 @@ s32 e1000_setup_init_funcs(struct e1000_hw *hw, bool init_device) case e1000_pchlan: case e1000_pch2lan: case e1000_pch_lpt: + case e1000_pch_spt: e1000_init_function_pointers_ich8lan(hw); break; case e1000_82575: diff --git a/freebsd/sys/dev/e1000/e1000_api.h b/freebsd/sys/dev/e1000/e1000_api.h index a2ffa169..074197bf 100644 --- a/freebsd/sys/dev/e1000/e1000_api.h +++ b/freebsd/sys/dev/e1000/e1000_api.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -124,14 +124,14 @@ u32 e1000_translate_register_82542(u32 reg); * TBI_ACCEPT macro definition: * * This macro requires: - * adapter = a pointer to struct e1000_hw + * a = a pointer to struct e1000_hw * status = the 8 bit status field of the Rx descriptor with EOP set - * error = the 8 bit error field of the Rx descriptor with EOP set + * errors = the 8 bit error field of the Rx descriptor with EOP set * length = the sum of all the length fields of the Rx descriptors that * make up the current frame * last_byte = the last byte of the frame DMAed by the hardware - * max_frame_length = the maximum frame length we want to accept. - * min_frame_length = the minimum frame length we want to accept. + * min_frame_size = the minimum frame length we want to accept. + * max_frame_size = the maximum frame length we want to accept. * * This macro is a conditional that should be used in the interrupt * handler's Rx processing routine when RxErrors have been detected. @@ -157,10 +157,10 @@ u32 e1000_translate_register_82542(u32 reg); (((errors) & E1000_RXD_ERR_FRAME_ERR_MASK) == E1000_RXD_ERR_CE) && \ ((last_byte) == CARRIER_EXTENSION) && \ (((status) & E1000_RXD_STAT_VP) ? \ - (((length) > (min_frame_size - VLAN_TAG_SIZE)) && \ - ((length) <= (max_frame_size + 1))) : \ - (((length) > min_frame_size) && \ - ((length) <= (max_frame_size + VLAN_TAG_SIZE + 1))))) + (((length) > ((min_frame_size) - VLAN_TAG_SIZE)) && \ + ((length) <= ((max_frame_size) + 1))) : \ + (((length) > (min_frame_size)) && \ + ((length) <= ((max_frame_size) + VLAN_TAG_SIZE + 1))))) #define E1000_MAX(a, b) ((a) > (b) ? (a) : (b)) #define E1000_DIVIDE_ROUND_UP(a, b) (((a) + (b) - 1) / (b)) /* ceil(a/b) */ diff --git a/freebsd/sys/dev/e1000/e1000_defines.h b/freebsd/sys/dev/e1000/e1000_defines.h index 72a8b14f..e33fe0fb 100644 --- a/freebsd/sys/dev/e1000/e1000_defines.h +++ b/freebsd/sys/dev/e1000/e1000_defines.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -158,11 +158,12 @@ E1000_RXDEXT_STATERR_CXE | \ E1000_RXDEXT_STATERR_RXE) -#define E1000_MRQC_ENABLE_RSS_2Q 0x00000001 +#define E1000_MRQC_RSS_ENABLE_2Q 0x00000001 #define E1000_MRQC_RSS_FIELD_MASK 0xFFFF0000 #define E1000_MRQC_RSS_FIELD_IPV4_TCP 0x00010000 #define E1000_MRQC_RSS_FIELD_IPV4 0x00020000 #define E1000_MRQC_RSS_FIELD_IPV6_TCP_EX 0x00040000 +#define E1000_MRQC_RSS_FIELD_IPV6_EX 0x00080000 #define E1000_MRQC_RSS_FIELD_IPV6 0x00100000 #define E1000_MRQC_RSS_FIELD_IPV6_TCP 0x00200000 @@ -196,6 +197,8 @@ #define E1000_RCTL_LBM_TCVR 0x000000C0 /* tcvr loopback mode */ #define E1000_RCTL_DTYP_PS 0x00000400 /* Packet Split descriptor */ #define E1000_RCTL_RDMTS_HALF 0x00000000 /* Rx desc min thresh size */ +#define E1000_RCTL_RDMTS_HEX 0x00010000 +#define E1000_RCTL_RDMTS1_HEX E1000_RCTL_RDMTS_HEX #define E1000_RCTL_MO_SHIFT 12 /* multicast offset shift */ #define E1000_RCTL_MO_3 0x00003000 /* multicast offset 15:4 */ #define E1000_RCTL_BAM 0x00008000 /* broadcast enable */ @@ -752,6 +755,12 @@ #define E1000_TSYNCTXCTL_VALID 0x00000001 /* Tx timestamp valid */ #define E1000_TSYNCTXCTL_ENABLED 0x00000010 /* enable Tx timestamping */ +/* HH Time Sync */ +#define E1000_TSYNCTXCTL_MAX_ALLOWED_DLY_MASK 0x0000F000 /* max delay */ +#define E1000_TSYNCTXCTL_SYNC_COMP_ERR 0x20000000 /* sync err */ +#define E1000_TSYNCTXCTL_SYNC_COMP 0x40000000 /* sync complete */ +#define E1000_TSYNCTXCTL_START_SYNC 0x80000000 /* initiate sync */ + #define E1000_TSYNCRXCTL_VALID 0x00000001 /* Rx timestamp valid */ #define E1000_TSYNCRXCTL_TYPE_MASK 0x0000000E /* Rx type mask */ #define E1000_TSYNCRXCTL_TYPE_L2_V2 0x00 @@ -848,6 +857,7 @@ #define E1000_M88E1543_PAGE_ADDR 0x16 /* Page Offset Register */ #define E1000_M88E1543_EEE_CTRL_1 0x0 #define E1000_M88E1543_EEE_CTRL_1_MS 0x0001 /* EEE Master/Slave */ +#define E1000_M88E1543_FIBER_CTRL 0x0 /* Fiber Control Register */ #define E1000_EEE_ADV_DEV_I354 7 #define E1000_EEE_ADV_ADDR_I354 60 #define E1000_EEE_ADV_100_SUPPORTED (1 << 1) /* 100BaseTx EEE Supported */ @@ -1019,9 +1029,7 @@ /* NVM Addressing bits based on type 0=small, 1=large */ #define E1000_EECD_ADDR_BITS 0x00000400 #define E1000_EECD_TYPE 0x00002000 /* NVM Type (1-SPI, 0-Microwire) */ -#ifndef E1000_NVM_GRANT_ATTEMPTS #define E1000_NVM_GRANT_ATTEMPTS 1000 /* NVM # attempts to gain grant */ -#endif #define E1000_EECD_AUTO_RD 0x00000200 /* NVM Auto Read done */ #define E1000_EECD_SIZE_EX_MASK 0x00007800 /* NVM Size */ #define E1000_EECD_SIZE_EX_SHIFT 11 diff --git a/freebsd/sys/dev/e1000/e1000_hw.h b/freebsd/sys/dev/e1000/e1000_hw.h index faf64a37..e1464a7b 100644 --- a/freebsd/sys/dev/e1000/e1000_hw.h +++ b/freebsd/sys/dev/e1000/e1000_hw.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -137,6 +137,15 @@ struct e1000_hw; #define E1000_DEV_ID_PCH_I218_V2 0x15A1 #define E1000_DEV_ID_PCH_I218_LM3 0x15A2 /* Wildcat Point PCH */ #define E1000_DEV_ID_PCH_I218_V3 0x15A3 /* Wildcat Point PCH */ +#define E1000_DEV_ID_PCH_SPT_I219_LM 0x156F /* Sunrise Point PCH */ +#define E1000_DEV_ID_PCH_SPT_I219_V 0x1570 /* Sunrise Point PCH */ +#define E1000_DEV_ID_PCH_SPT_I219_LM2 0x15B7 /* Sunrise Point-H PCH */ +#define E1000_DEV_ID_PCH_SPT_I219_V2 0x15B8 /* Sunrise Point-H PCH */ +#define E1000_DEV_ID_PCH_LBG_I219_LM3 0x15B9 /* LEWISBURG PCH */ +#define E1000_DEV_ID_PCH_SPT_I219_LM4 0x15D7 +#define E1000_DEV_ID_PCH_SPT_I219_V4 0x15D8 +#define E1000_DEV_ID_PCH_SPT_I219_LM5 0x15E3 +#define E1000_DEV_ID_PCH_SPT_I219_V5 0x15D6 #define E1000_DEV_ID_82576 0x10C9 #define E1000_DEV_ID_82576_FIBER 0x10E6 #define E1000_DEV_ID_82576_SERDES 0x10E7 @@ -222,6 +231,7 @@ enum e1000_mac_type { e1000_pchlan, e1000_pch2lan, e1000_pch_lpt, + e1000_pch_spt, e1000_82575, e1000_82576, e1000_82580, @@ -787,7 +797,7 @@ struct e1000_mac_info { u16 uta_reg_count; /* Maximum size of the MTA register table in all supported adapters */ - #define MAX_MTA_REG 128 +#define MAX_MTA_REG 128 u32 mta_shadow[MAX_MTA_REG]; u16 rar_entry_count; @@ -805,7 +815,7 @@ struct e1000_mac_info { enum e1000_serdes_link_state serdes_link_state; bool serdes_has_link; bool tx_pkt_filtering; - u32 max_frame_size; + u32 max_frame_size; }; struct e1000_phy_info { @@ -951,9 +961,13 @@ struct e1000_dev_spec_ich8lan { E1000_MUTEX nvm_mutex; E1000_MUTEX swflag_mutex; bool nvm_k1_enabled; + bool disable_k1_off; bool eee_disable; u16 eee_lp_ability; enum e1000_ulp_state ulp_state; + bool ulp_capability_disabled; + bool during_suspend_flow; + bool during_dpg_exit; }; struct e1000_dev_spec_82575 { diff --git a/freebsd/sys/dev/e1000/e1000_i210.c b/freebsd/sys/dev/e1000/e1000_i210.c index 413b6a1d..8f427b9a 100644 --- a/freebsd/sys/dev/e1000/e1000_i210.c +++ b/freebsd/sys/dev/e1000/e1000_i210.c @@ -2,7 +2,7 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -885,6 +885,35 @@ static s32 e1000_pll_workaround_i210(struct e1000_hw *hw) } /** + * e1000_get_cfg_done_i210 - Read config done bit + * @hw: pointer to the HW structure + * + * Read the management control register for the config done bit for + * completion status. NOTE: silicon which is EEPROM-less will fail trying + * to read the config done bit, so an error is *ONLY* logged and returns + * E1000_SUCCESS. If we were to return with error, EEPROM-less silicon + * would not be able to be reset or change link. + **/ +static s32 e1000_get_cfg_done_i210(struct e1000_hw *hw) +{ + s32 timeout = PHY_CFG_TIMEOUT; + u32 mask = E1000_NVM_CFG_DONE_PORT_0; + + DEBUGFUNC("e1000_get_cfg_done_i210"); + + while (timeout) { + if (E1000_READ_REG(hw, E1000_EEMNGCTL_I210) & mask) + break; + msec_delay(1); + timeout--; + } + if (!timeout) + DEBUGOUT("MNG configuration cycle has not completed.\n"); + + return E1000_SUCCESS; +} + +/** * e1000_init_hw_i210 - Init hw for I210/I211 * @hw: pointer to the HW structure * @@ -901,6 +930,7 @@ s32 e1000_init_hw_i210(struct e1000_hw *hw) if (ret_val != E1000_SUCCESS) return ret_val; } + hw->phy.ops.get_cfg_done = e1000_get_cfg_done_i210; ret_val = e1000_init_hw_82575(hw); return ret_val; } diff --git a/freebsd/sys/dev/e1000/e1000_i210.h b/freebsd/sys/dev/e1000/e1000_i210.h index 2a20ca1e..f940915b 100644 --- a/freebsd/sys/dev/e1000/e1000_i210.h +++ b/freebsd/sys/dev/e1000/e1000_i210.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/freebsd/sys/dev/e1000/e1000_ich8lan.c b/freebsd/sys/dev/e1000/e1000_ich8lan.c index b5c75f26..007488b2 100644 --- a/freebsd/sys/dev/e1000/e1000_ich8lan.c +++ b/freebsd/sys/dev/e1000/e1000_ich8lan.c @@ -2,7 +2,7 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -94,10 +94,13 @@ static s32 e1000_set_d3_lplu_state_ich8lan(struct e1000_hw *hw, bool active); static s32 e1000_read_nvm_ich8lan(struct e1000_hw *hw, u16 offset, u16 words, u16 *data); +static s32 e1000_read_nvm_spt(struct e1000_hw *hw, u16 offset, u16 words, + u16 *data); static s32 e1000_write_nvm_ich8lan(struct e1000_hw *hw, u16 offset, u16 words, u16 *data); static s32 e1000_validate_nvm_checksum_ich8lan(struct e1000_hw *hw); static s32 e1000_update_nvm_checksum_ich8lan(struct e1000_hw *hw); +static s32 e1000_update_nvm_checksum_spt(struct e1000_hw *hw); static s32 e1000_valid_led_default_ich8lan(struct e1000_hw *hw, u16 *data); static s32 e1000_id_led_init_pchlan(struct e1000_hw *hw); @@ -125,6 +128,14 @@ static s32 e1000_read_flash_byte_ich8lan(struct e1000_hw *hw, u32 offset, u8 *data); static s32 e1000_read_flash_data_ich8lan(struct e1000_hw *hw, u32 offset, u8 size, u16 *data); +static s32 e1000_read_flash_data32_ich8lan(struct e1000_hw *hw, u32 offset, + u32 *data); +static s32 e1000_read_flash_dword_ich8lan(struct e1000_hw *hw, + u32 offset, u32 *data); +static s32 e1000_write_flash_data32_ich8lan(struct e1000_hw *hw, + u32 offset, u32 data); +static s32 e1000_retry_write_flash_dword_ich8lan(struct e1000_hw *hw, + u32 offset, u32 dword); static s32 e1000_read_flash_word_ich8lan(struct e1000_hw *hw, u32 offset, u16 *data); static s32 e1000_retry_write_flash_byte_ich8lan(struct e1000_hw *hw, @@ -234,16 +245,21 @@ static bool e1000_phy_is_accessible_pchlan(struct e1000_hw *hw) if (ret_val) return FALSE; out: - if (hw->mac.type == e1000_pch_lpt) { - /* Unforce SMBus mode in PHY */ - hw->phy.ops.read_reg_locked(hw, CV_SMB_CTRL, &phy_reg); - phy_reg &= ~CV_SMB_CTRL_FORCE_SMBUS; - hw->phy.ops.write_reg_locked(hw, CV_SMB_CTRL, phy_reg); + if ((hw->mac.type == e1000_pch_lpt) || + (hw->mac.type == e1000_pch_spt)) { + /* Only unforce SMBus if ME is not active */ + if (!(E1000_READ_REG(hw, E1000_FWSM) & + E1000_ICH_FWSM_FW_VALID)) { + /* Unforce SMBus mode in PHY */ + hw->phy.ops.read_reg_locked(hw, CV_SMB_CTRL, &phy_reg); + phy_reg &= ~CV_SMB_CTRL_FORCE_SMBUS; + hw->phy.ops.write_reg_locked(hw, CV_SMB_CTRL, phy_reg); - /* Unforce SMBus mode in MAC */ - mac_reg = E1000_READ_REG(hw, E1000_CTRL_EXT); - mac_reg &= ~E1000_CTRL_EXT_FORCE_SMBUS; - E1000_WRITE_REG(hw, E1000_CTRL_EXT, mac_reg); + /* Unforce SMBus mode in MAC */ + mac_reg = E1000_READ_REG(hw, E1000_CTRL_EXT); + mac_reg &= ~E1000_CTRL_EXT_FORCE_SMBUS; + E1000_WRITE_REG(hw, E1000_CTRL_EXT, mac_reg); + } } return TRUE; @@ -274,7 +290,7 @@ static void e1000_toggle_lanphypc_pch_lpt(struct e1000_hw *hw) mac_reg &= ~E1000_CTRL_LANPHYPC_VALUE; E1000_WRITE_REG(hw, E1000_CTRL, mac_reg); E1000_WRITE_FLUSH(hw); - usec_delay(10); + msec_delay(1); mac_reg &= ~E1000_CTRL_LANPHYPC_OVERRIDE; E1000_WRITE_REG(hw, E1000_CTRL, mac_reg); E1000_WRITE_FLUSH(hw); @@ -330,6 +346,7 @@ static s32 e1000_init_phy_workarounds_pchlan(struct e1000_hw *hw) */ switch (hw->mac.type) { case e1000_pch_lpt: + case e1000_pch_spt: if (e1000_phy_is_accessible_pchlan(hw)) break; @@ -477,6 +494,7 @@ static s32 e1000_init_phy_params_pchlan(struct e1000_hw *hw) /* fall-through */ case e1000_pch2lan: case e1000_pch_lpt: + case e1000_pch_spt: /* In case the PHY needs to be in mdio slow mode, * set slow mode and try to get the PHY id again. */ @@ -619,36 +637,57 @@ static s32 e1000_init_nvm_params_ich8lan(struct e1000_hw *hw) struct e1000_dev_spec_ich8lan *dev_spec = &hw->dev_spec.ich8lan; u32 gfpreg, sector_base_addr, sector_end_addr; u16 i; + u32 nvm_size; DEBUGFUNC("e1000_init_nvm_params_ich8lan"); - /* Can't read flash registers if the register set isn't mapped. */ nvm->type = e1000_nvm_flash_sw; - if (!hw->flash_address) { - DEBUGOUT("ERROR: Flash registers not mapped\n"); - return -E1000_ERR_CONFIG; - } - gfpreg = E1000_READ_FLASH_REG(hw, ICH_FLASH_GFPREG); + if (hw->mac.type == e1000_pch_spt) { + /* in SPT, gfpreg doesn't exist. NVM size is taken from the + * STRAP register. This is because in SPT the GbE Flash region + * is no longer accessed through the flash registers. Instead, + * the mechanism has changed, and the Flash region access + * registers are now implemented in GbE memory space. + */ + nvm->flash_base_addr = 0; + nvm_size = + (((E1000_READ_REG(hw, E1000_STRAP) >> 1) & 0x1F) + 1) + * NVM_SIZE_MULTIPLIER; + nvm->flash_bank_size = nvm_size / 2; + /* Adjust to word count */ + nvm->flash_bank_size /= sizeof(u16); + /* Set the base address for flash register access */ + hw->flash_address = hw->hw_addr + E1000_FLASH_BASE_ADDR; + } else { + /* Can't read flash registers if register set isn't mapped. */ + if (!hw->flash_address) { + DEBUGOUT("ERROR: Flash registers not mapped\n"); + return -E1000_ERR_CONFIG; + } - /* sector_X_addr is a "sector"-aligned address (4096 bytes) - * Add 1 to sector_end_addr since this sector is included in - * the overall size. - */ - sector_base_addr = gfpreg & FLASH_GFPREG_BASE_MASK; - sector_end_addr = ((gfpreg >> 16) & FLASH_GFPREG_BASE_MASK) + 1; + gfpreg = E1000_READ_FLASH_REG(hw, ICH_FLASH_GFPREG); - /* flash_base_addr is byte-aligned */ - nvm->flash_base_addr = sector_base_addr << FLASH_SECTOR_ADDR_SHIFT; + /* sector_X_addr is a "sector"-aligned address (4096 bytes) + * Add 1 to sector_end_addr since this sector is included in + * the overall size. + */ + sector_base_addr = gfpreg & FLASH_GFPREG_BASE_MASK; + sector_end_addr = ((gfpreg >> 16) & FLASH_GFPREG_BASE_MASK) + 1; - /* find total size of the NVM, then cut in half since the total - * size represents two separate NVM banks. - */ - nvm->flash_bank_size = ((sector_end_addr - sector_base_addr) - << FLASH_SECTOR_ADDR_SHIFT); - nvm->flash_bank_size /= 2; - /* Adjust to word count */ - nvm->flash_bank_size /= sizeof(u16); + /* flash_base_addr is byte-aligned */ + nvm->flash_base_addr = sector_base_addr + << FLASH_SECTOR_ADDR_SHIFT; + + /* find total size of the NVM, then cut in half since the total + * size represents two separate NVM banks. + */ + nvm->flash_bank_size = ((sector_end_addr - sector_base_addr) + << FLASH_SECTOR_ADDR_SHIFT); + nvm->flash_bank_size /= 2; + /* Adjust to word count */ + nvm->flash_bank_size /= sizeof(u16); + } nvm->word_size = E1000_SHADOW_RAM_WORDS; @@ -664,8 +703,13 @@ static s32 e1000_init_nvm_params_ich8lan(struct e1000_hw *hw) /* Function Pointers */ nvm->ops.acquire = e1000_acquire_nvm_ich8lan; nvm->ops.release = e1000_release_nvm_ich8lan; - nvm->ops.read = e1000_read_nvm_ich8lan; - nvm->ops.update = e1000_update_nvm_checksum_ich8lan; + if (hw->mac.type == e1000_pch_spt) { + nvm->ops.read = e1000_read_nvm_spt; + nvm->ops.update = e1000_update_nvm_checksum_spt; + } else { + nvm->ops.read = e1000_read_nvm_ich8lan; + nvm->ops.update = e1000_update_nvm_checksum_ich8lan; + } nvm->ops.valid_led_default = e1000_valid_led_default_ich8lan; nvm->ops.validate = e1000_validate_nvm_checksum_ich8lan; nvm->ops.write = e1000_write_nvm_ich8lan; @@ -751,9 +795,11 @@ static s32 e1000_init_mac_params_ich8lan(struct e1000_hw *hw) mac->ops.rar_set = e1000_rar_set_pch2lan; /* fall-through */ case e1000_pch_lpt: + case e1000_pch_spt: /* multicast address update for pch2 */ mac->ops.update_mc_addr_list = e1000_update_mc_addr_list_pch2lan; + /* fall-through */ case e1000_pchlan: /* check management mode */ mac->ops.check_mng_mode = e1000_check_mng_mode_pchlan; @@ -771,7 +817,8 @@ static s32 e1000_init_mac_params_ich8lan(struct e1000_hw *hw) break; } - if (mac->type == e1000_pch_lpt) { + if ((mac->type == e1000_pch_lpt) || + (mac->type == e1000_pch_spt)) { mac->rar_entry_count = E1000_PCH_LPT_RAR_ENTRIES; mac->ops.rar_set = e1000_rar_set_pch_lpt; mac->ops.setup_physical_interface = e1000_setup_copper_link_pch_lpt; @@ -1001,8 +1048,9 @@ release: /* clear FEXTNVM6 bit 8 on link down or 10/100 */ fextnvm6 &= ~E1000_FEXTNVM6_REQ_PLL_CLK; - if (!link || ((status & E1000_STATUS_SPEED_100) && - (status & E1000_STATUS_FD))) + if ((hw->phy.revision > 5) || !link || + ((status & E1000_STATUS_SPEED_100) && + (status & E1000_STATUS_FD))) goto update_fextnvm6; ret_val = hw->phy.ops.read_reg(hw, I217_INBAND_CTRL, ®); @@ -1081,7 +1129,7 @@ static s32 e1000_platform_pm_pch_lpt(struct e1000_hw *hw, bool link) u16 speed, duplex, scale = 0; u16 max_snoop, max_nosnoop; u16 max_ltr_enc; /* max LTR latency encoded */ - s64 lat_ns; /* latency (ns) */ + s64 lat_ns; s64 value; u32 rxa; @@ -1113,8 +1161,8 @@ static s32 e1000_platform_pm_pch_lpt(struct e1000_hw *hw, bool link) lat_ns = 0; else lat_ns /= speed; - value = lat_ns; + while (value > E1000_LTRV_VALUE_MASK) { scale++; value = E1000_DIVIDE_ROUND_UP(value, (1 << 5)); @@ -1215,6 +1263,7 @@ s32 e1000_enable_ulp_lpt_lp(struct e1000_hw *hw, bool to_sx) u32 mac_reg; s32 ret_val = E1000_SUCCESS; u16 phy_reg; + u16 oem_reg = 0; if ((hw->mac.type < e1000_pch_lpt) || (hw->device_id == E1000_DEV_ID_PCH_LPT_I217_LM) || @@ -1270,6 +1319,25 @@ s32 e1000_enable_ulp_lpt_lp(struct e1000_hw *hw, bool to_sx) mac_reg |= E1000_CTRL_EXT_FORCE_SMBUS; E1000_WRITE_REG(hw, E1000_CTRL_EXT, mac_reg); + /* Si workaround for ULP entry flow on i127/rev6 h/w. Enable + * LPLU and disable Gig speed when entering ULP + */ + if ((hw->phy.type == e1000_phy_i217) && (hw->phy.revision == 6)) { + ret_val = e1000_read_phy_reg_hv_locked(hw, HV_OEM_BITS, + &oem_reg); + if (ret_val) + goto release; + + phy_reg = oem_reg; + phy_reg |= HV_OEM_BITS_LPLU | HV_OEM_BITS_GBE_DIS; + + ret_val = e1000_write_phy_reg_hv_locked(hw, HV_OEM_BITS, + phy_reg); + + if (ret_val) + goto release; + } + /* Set Inband ULP Exit, Reset to SMBus mode and * Disable SMBus Release on PERST# in PHY */ @@ -1281,10 +1349,15 @@ s32 e1000_enable_ulp_lpt_lp(struct e1000_hw *hw, bool to_sx) if (to_sx) { if (E1000_READ_REG(hw, E1000_WUFC) & E1000_WUFC_LNKC) phy_reg |= I218_ULP_CONFIG1_WOL_HOST; + else + phy_reg &= ~I218_ULP_CONFIG1_WOL_HOST; phy_reg |= I218_ULP_CONFIG1_STICKY_ULP; + phy_reg &= ~I218_ULP_CONFIG1_INBAND_EXIT; } else { phy_reg |= I218_ULP_CONFIG1_INBAND_EXIT; + phy_reg &= ~I218_ULP_CONFIG1_STICKY_ULP; + phy_reg &= ~I218_ULP_CONFIG1_WOL_HOST; } e1000_write_phy_reg_hv_locked(hw, I218_ULP_CONFIG1, phy_reg); @@ -1296,12 +1369,21 @@ s32 e1000_enable_ulp_lpt_lp(struct e1000_hw *hw, bool to_sx) /* Commit ULP changes in PHY by starting auto ULP configuration */ phy_reg |= I218_ULP_CONFIG1_START; e1000_write_phy_reg_hv_locked(hw, I218_ULP_CONFIG1, phy_reg); + + if ((hw->phy.type == e1000_phy_i217) && (hw->phy.revision == 6) && + to_sx && (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) { + ret_val = e1000_write_phy_reg_hv_locked(hw, HV_OEM_BITS, + oem_reg); + if (ret_val) + goto release; + } + release: hw->phy.ops.release(hw); out: - if (ret_val) { + if (ret_val) DEBUGOUT1("Error in ULP enable flow: %d\n", ret_val); - } else + else hw->dev_spec.ich8lan.ulp_state = e1000_ulp_state_on; return ret_val; @@ -1346,10 +1428,10 @@ s32 e1000_disable_ulp_lpt_lp(struct e1000_hw *hw, bool force) E1000_WRITE_REG(hw, E1000_H2ME, mac_reg); } - /* Poll up to 100msec for ME to clear ULP_CFG_DONE */ + /* Poll up to 300msec for ME to clear ULP_CFG_DONE. */ while (E1000_READ_REG(hw, E1000_FWSM) & E1000_FWSM_ULP_CFG_DONE) { - if (i++ == 10) { + if (i++ == 30) { ret_val = -E1000_ERR_PHY; goto out; } @@ -1423,6 +1505,8 @@ s32 e1000_disable_ulp_lpt_lp(struct e1000_hw *hw, bool force) I218_ULP_CONFIG1_RESET_TO_SMBUS | I218_ULP_CONFIG1_WOL_HOST | I218_ULP_CONFIG1_INBAND_EXIT | + I218_ULP_CONFIG1_EN_ULP_LANPHYPC | + I218_ULP_CONFIG1_DIS_CLR_STICKY_ON_PERST | I218_ULP_CONFIG1_DISABLE_SMB_PERST); e1000_write_phy_reg_hv_locked(hw, I218_ULP_CONFIG1, phy_reg); @@ -1442,9 +1526,9 @@ release: msec_delay(50); } out: - if (ret_val) { + if (ret_val) DEBUGOUT1("Error in ULP disable flow: %d\n", ret_val); - } else + else hw->dev_spec.ich8lan.ulp_state = e1000_ulp_state_off; return ret_val; @@ -1461,7 +1545,8 @@ out: static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw) { struct e1000_mac_info *mac = &hw->mac; - s32 ret_val; + s32 ret_val, tipg_reg = 0; + u16 emi_addr, emi_val = 0; bool link; u16 phy_reg; @@ -1494,35 +1579,129 @@ static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw) * the IPG and reduce Rx latency in the PHY. */ if (((hw->mac.type == e1000_pch2lan) || - (hw->mac.type == e1000_pch_lpt)) && link) { - u32 reg; - reg = E1000_READ_REG(hw, E1000_STATUS); - if (!(reg & (E1000_STATUS_FD | E1000_STATUS_SPEED_MASK))) { - u16 emi_addr; + (hw->mac.type == e1000_pch_lpt) || + (hw->mac.type == e1000_pch_spt)) && link) { + u16 speed, duplex; - reg = E1000_READ_REG(hw, E1000_TIPG); - reg &= ~E1000_TIPG_IPGT_MASK; - reg |= 0xFF; - E1000_WRITE_REG(hw, E1000_TIPG, reg); + e1000_get_speed_and_duplex_copper_generic(hw, &speed, &duplex); + tipg_reg = E1000_READ_REG(hw, E1000_TIPG); + tipg_reg &= ~E1000_TIPG_IPGT_MASK; + if (duplex == HALF_DUPLEX && speed == SPEED_10) { + tipg_reg |= 0xFF; /* Reduce Rx latency in analog PHY */ - ret_val = hw->phy.ops.acquire(hw); - if (ret_val) - return ret_val; + emi_val = 0; + } else if (hw->mac.type == e1000_pch_spt && + duplex == FULL_DUPLEX && speed != SPEED_1000) { + tipg_reg |= 0xC; + emi_val = 1; + } else { + /* Roll back the default values */ + tipg_reg |= 0x08; + emi_val = 1; + } - if (hw->mac.type == e1000_pch2lan) - emi_addr = I82579_RX_CONFIG; + E1000_WRITE_REG(hw, E1000_TIPG, tipg_reg); + + ret_val = hw->phy.ops.acquire(hw); + if (ret_val) + return ret_val; + + if (hw->mac.type == e1000_pch2lan) + emi_addr = I82579_RX_CONFIG; + else + emi_addr = I217_RX_CONFIG; + ret_val = e1000_write_emi_reg_locked(hw, emi_addr, emi_val); + + if (hw->mac.type == e1000_pch_lpt || + hw->mac.type == e1000_pch_spt) { + u16 phy_reg; + + hw->phy.ops.read_reg_locked(hw, I217_PLL_CLOCK_GATE_REG, + &phy_reg); + phy_reg &= ~I217_PLL_CLOCK_GATE_MASK; + if (speed == SPEED_100 || speed == SPEED_10) + phy_reg |= 0x3E8; else - emi_addr = I217_RX_CONFIG; - ret_val = e1000_write_emi_reg_locked(hw, emi_addr, 0); + phy_reg |= 0xFA; + hw->phy.ops.write_reg_locked(hw, + I217_PLL_CLOCK_GATE_REG, + phy_reg); - hw->phy.ops.release(hw); + if (speed == SPEED_1000) { + hw->phy.ops.read_reg_locked(hw, HV_PM_CTRL, + &phy_reg); - if (ret_val) - return ret_val; + phy_reg |= HV_PM_CTRL_K1_CLK_REQ; + + hw->phy.ops.write_reg_locked(hw, HV_PM_CTRL, + phy_reg); + } + } + hw->phy.ops.release(hw); + + if (ret_val) + return ret_val; + + if (hw->mac.type == e1000_pch_spt) { + u16 data; + u16 ptr_gap; + + if (speed == SPEED_1000) { + ret_val = hw->phy.ops.acquire(hw); + if (ret_val) + return ret_val; + + ret_val = hw->phy.ops.read_reg_locked(hw, + PHY_REG(776, 20), + &data); + if (ret_val) { + hw->phy.ops.release(hw); + return ret_val; + } + + ptr_gap = (data & (0x3FF << 2)) >> 2; + if (ptr_gap < 0x18) { + data &= ~(0x3FF << 2); + data |= (0x18 << 2); + ret_val = + hw->phy.ops.write_reg_locked(hw, + PHY_REG(776, 20), data); + } + hw->phy.ops.release(hw); + if (ret_val) + return ret_val; + } else { + ret_val = hw->phy.ops.acquire(hw); + if (ret_val) + return ret_val; + + ret_val = hw->phy.ops.write_reg_locked(hw, + PHY_REG(776, 20), + 0xC023); + hw->phy.ops.release(hw); + if (ret_val) + return ret_val; + + } } } + /* I217 Packet Loss issue: + * ensure that FEXTNVM4 Beacon Duration is set correctly + * on power up. + * Set the Beacon Duration for I217 to 8 usec + */ + if ((hw->mac.type == e1000_pch_lpt) || + (hw->mac.type == e1000_pch_spt)) { + u32 mac_reg; + + mac_reg = E1000_READ_REG(hw, E1000_FEXTNVM4); + mac_reg &= ~E1000_FEXTNVM4_BEACON_DURATION_MASK; + mac_reg |= E1000_FEXTNVM4_BEACON_DURATION_8USEC; + E1000_WRITE_REG(hw, E1000_FEXTNVM4, mac_reg); + } + /* Work-around I218 hang issue */ if ((hw->device_id == E1000_DEV_ID_PCH_LPTLP_I218_LM) || (hw->device_id == E1000_DEV_ID_PCH_LPTLP_I218_V) || @@ -1532,7 +1711,8 @@ static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw) if (ret_val) return ret_val; } - if (hw->mac.type == e1000_pch_lpt) { + if ((hw->mac.type == e1000_pch_lpt) || + (hw->mac.type == e1000_pch_spt)) { /* Set platform power management values for * Latency Tolerance Reporting (LTR) * Optimized Buffer Flush/Fill (OBFF) @@ -1545,6 +1725,20 @@ static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw) /* Clear link partner's EEE ability */ hw->dev_spec.ich8lan.eee_lp_ability = 0; + /* FEXTNVM6 K1-off workaround */ + if (hw->mac.type == e1000_pch_spt) { + u32 pcieanacfg = E1000_READ_REG(hw, E1000_PCIEANACFG); + u32 fextnvm6 = E1000_READ_REG(hw, E1000_FEXTNVM6); + + if ((pcieanacfg & E1000_FEXTNVM6_K1_OFF_ENABLE) && + (hw->dev_spec.ich8lan.disable_k1_off == FALSE)) + fextnvm6 |= E1000_FEXTNVM6_K1_OFF_ENABLE; + else + fextnvm6 &= ~E1000_FEXTNVM6_K1_OFF_ENABLE; + + E1000_WRITE_REG(hw, E1000_FEXTNVM6, fextnvm6); + } + if (!link) return E1000_SUCCESS; /* No link detected */ @@ -1638,6 +1832,7 @@ void e1000_init_function_pointers_ich8lan(struct e1000_hw *hw) case e1000_pchlan: case e1000_pch2lan: case e1000_pch_lpt: + case e1000_pch_spt: hw->phy.ops.init_params = e1000_init_phy_params_pchlan; break; default: @@ -2020,7 +2215,7 @@ static s32 e1000_check_reset_block_ich8lan(struct e1000_hw *hw) continue; } blocked = FALSE; - } while (blocked && (i++ < 10)); + } while (blocked && (i++ < 30)); return blocked ? E1000_BLK_PHY_RESET : E1000_SUCCESS; } @@ -2101,6 +2296,7 @@ static s32 e1000_sw_lcd_config_ich8lan(struct e1000_hw *hw) case e1000_pchlan: case e1000_pch2lan: case e1000_pch_lpt: + case e1000_pch_spt: sw_cfg_mask = E1000_FEXTNVM_SW_CONFIG_ICH8M; break; default: @@ -2991,7 +3187,6 @@ static s32 e1000_set_lplu_state_pchlan(struct e1000_hw *hw, bool active) u16 oem_reg; DEBUGFUNC("e1000_set_lplu_state_pchlan"); - ret_val = hw->phy.ops.read_reg(hw, HV_OEM_BITS, &oem_reg); if (ret_val) return ret_val; @@ -3211,12 +3406,47 @@ static s32 e1000_valid_nvm_bank_detect_ich8lan(struct e1000_hw *hw, u32 *bank) struct e1000_nvm_info *nvm = &hw->nvm; u32 bank1_offset = nvm->flash_bank_size * sizeof(u16); u32 act_offset = E1000_ICH_NVM_SIG_WORD * 2 + 1; + u32 nvm_dword = 0; u8 sig_byte = 0; s32 ret_val; DEBUGFUNC("e1000_valid_nvm_bank_detect_ich8lan"); switch (hw->mac.type) { + case e1000_pch_spt: + bank1_offset = nvm->flash_bank_size; + act_offset = E1000_ICH_NVM_SIG_WORD; + + /* set bank to 0 in case flash read fails */ + *bank = 0; + + /* Check bank 0 */ + ret_val = e1000_read_flash_dword_ich8lan(hw, act_offset, + &nvm_dword); + if (ret_val) + return ret_val; + sig_byte = (u8)((nvm_dword & 0xFF00) >> 8); + if ((sig_byte & E1000_ICH_NVM_VALID_SIG_MASK) == + E1000_ICH_NVM_SIG_VALUE) { + *bank = 0; + return E1000_SUCCESS; + } + + /* Check bank 1 */ + ret_val = e1000_read_flash_dword_ich8lan(hw, act_offset + + bank1_offset, + &nvm_dword); + if (ret_val) + return ret_val; + sig_byte = (u8)((nvm_dword & 0xFF00) >> 8); + if ((sig_byte & E1000_ICH_NVM_VALID_SIG_MASK) == + E1000_ICH_NVM_SIG_VALUE) { + *bank = 1; + return E1000_SUCCESS; + } + + DEBUGOUT("ERROR: No valid NVM bank present\n"); + return -E1000_ERR_NVM; case e1000_ich8lan: case e1000_ich9lan: eecd = E1000_READ_REG(hw, E1000_EECD); @@ -3264,6 +3494,99 @@ static s32 e1000_valid_nvm_bank_detect_ich8lan(struct e1000_hw *hw, u32 *bank) } /** + * e1000_read_nvm_spt - NVM access for SPT + * @hw: pointer to the HW structure + * @offset: The offset (in bytes) of the word(s) to read. + * @words: Size of data to read in words. + * @data: pointer to the word(s) to read at offset. + * + * Reads a word(s) from the NVM + **/ +static s32 e1000_read_nvm_spt(struct e1000_hw *hw, u16 offset, u16 words, + u16 *data) +{ + struct e1000_nvm_info *nvm = &hw->nvm; + struct e1000_dev_spec_ich8lan *dev_spec = &hw->dev_spec.ich8lan; + u32 act_offset; + s32 ret_val = E1000_SUCCESS; + u32 bank = 0; + u32 dword = 0; + u16 offset_to_read; + u16 i; + + DEBUGFUNC("e1000_read_nvm_spt"); + + if ((offset >= nvm->word_size) || (words > nvm->word_size - offset) || + (words == 0)) { + DEBUGOUT("nvm parameter(s) out of bounds\n"); + ret_val = -E1000_ERR_NVM; + goto out; + } + + nvm->ops.acquire(hw); + + ret_val = e1000_valid_nvm_bank_detect_ich8lan(hw, &bank); + if (ret_val != E1000_SUCCESS) { + DEBUGOUT("Could not detect valid bank, assuming bank 0\n"); + bank = 0; + } + + act_offset = (bank) ? nvm->flash_bank_size : 0; + act_offset += offset; + + ret_val = E1000_SUCCESS; + + for (i = 0; i < words; i += 2) { + if (words - i == 1) { + if (dev_spec->shadow_ram[offset+i].modified) { + data[i] = dev_spec->shadow_ram[offset+i].value; + } else { + offset_to_read = act_offset + i - + ((act_offset + i) % 2); + ret_val = + e1000_read_flash_dword_ich8lan(hw, + offset_to_read, + &dword); + if (ret_val) + break; + if ((act_offset + i) % 2 == 0) + data[i] = (u16)(dword & 0xFFFF); + else + data[i] = (u16)((dword >> 16) & 0xFFFF); + } + } else { + offset_to_read = act_offset + i; + if (!(dev_spec->shadow_ram[offset+i].modified) || + !(dev_spec->shadow_ram[offset+i+1].modified)) { + ret_val = + e1000_read_flash_dword_ich8lan(hw, + offset_to_read, + &dword); + if (ret_val) + break; + } + if (dev_spec->shadow_ram[offset+i].modified) + data[i] = dev_spec->shadow_ram[offset+i].value; + else + data[i] = (u16) (dword & 0xFFFF); + if (dev_spec->shadow_ram[offset+i].modified) + data[i+1] = + dev_spec->shadow_ram[offset+i+1].value; + else + data[i+1] = (u16) (dword >> 16 & 0xFFFF); + } + } + + nvm->ops.release(hw); + +out: + if (ret_val) + DEBUGOUT1("NVM read error: %d\n", ret_val); + + return ret_val; +} + +/** * e1000_read_nvm_ich8lan - Read word(s) from the NVM * @hw: pointer to the HW structure * @offset: The offset (in bytes) of the word(s) to read. @@ -3350,7 +3673,11 @@ static s32 e1000_flash_cycle_init_ich8lan(struct e1000_hw *hw) /* Clear FCERR and DAEL in hw status by writing 1 */ hsfsts.hsf_status.flcerr = 1; hsfsts.hsf_status.dael = 1; - E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFSTS, hsfsts.regval); + if (hw->mac.type == e1000_pch_spt) + E1000_WRITE_FLASH_REG(hw, ICH_FLASH_HSFSTS, + hsfsts.regval & 0xFFFF); + else + E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFSTS, hsfsts.regval); /* Either we should have a hardware SPI cycle in progress * bit to check against, in order to start a new cycle or @@ -3366,7 +3693,12 @@ static s32 e1000_flash_cycle_init_ich8lan(struct e1000_hw *hw) * Begin by setting Flash Cycle Done. */ hsfsts.hsf_status.flcdone = 1; - E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFSTS, hsfsts.regval); + if (hw->mac.type == e1000_pch_spt) + E1000_WRITE_FLASH_REG(hw, ICH_FLASH_HSFSTS, + hsfsts.regval & 0xFFFF); + else + E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFSTS, + hsfsts.regval); ret_val = E1000_SUCCESS; } else { s32 i; @@ -3388,8 +3720,12 @@ static s32 e1000_flash_cycle_init_ich8lan(struct e1000_hw *hw) * now set the Flash Cycle Done. */ hsfsts.hsf_status.flcdone = 1; - E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFSTS, - hsfsts.regval); + if (hw->mac.type == e1000_pch_spt) + E1000_WRITE_FLASH_REG(hw, ICH_FLASH_HSFSTS, + hsfsts.regval & 0xFFFF); + else + E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFSTS, + hsfsts.regval); } else { DEBUGOUT("Flash controller busy, cannot get access\n"); } @@ -3414,10 +3750,17 @@ static s32 e1000_flash_cycle_ich8lan(struct e1000_hw *hw, u32 timeout) DEBUGFUNC("e1000_flash_cycle_ich8lan"); /* Start a cycle by writing 1 in Flash Cycle Go in Hw Flash Control */ - hsflctl.regval = E1000_READ_FLASH_REG16(hw, ICH_FLASH_HSFCTL); + if (hw->mac.type == e1000_pch_spt) + hsflctl.regval = E1000_READ_FLASH_REG(hw, ICH_FLASH_HSFSTS)>>16; + else + hsflctl.regval = E1000_READ_FLASH_REG16(hw, ICH_FLASH_HSFCTL); hsflctl.hsf_ctrl.flcgo = 1; - E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFCTL, hsflctl.regval); + if (hw->mac.type == e1000_pch_spt) + E1000_WRITE_FLASH_REG(hw, ICH_FLASH_HSFSTS, + hsflctl.regval << 16); + else + E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFCTL, hsflctl.regval); /* wait till FDONE bit is set to 1 */ do { @@ -3434,6 +3777,29 @@ static s32 e1000_flash_cycle_ich8lan(struct e1000_hw *hw, u32 timeout) } /** + * e1000_read_flash_dword_ich8lan - Read dword from flash + * @hw: pointer to the HW structure + * @offset: offset to data location + * @data: pointer to the location for storing the data + * + * Reads the flash dword at offset into data. Offset is converted + * to bytes before read. + **/ +static s32 e1000_read_flash_dword_ich8lan(struct e1000_hw *hw, u32 offset, + u32 *data) +{ + DEBUGFUNC("e1000_read_flash_dword_ich8lan"); + + if (!data) + return -E1000_ERR_NVM; + + /* Must convert word offset into bytes. */ + offset <<= 1; + + return e1000_read_flash_data32_ich8lan(hw, offset, data); +} + +/** * e1000_read_flash_word_ich8lan - Read word from flash * @hw: pointer to the HW structure * @offset: offset to data location @@ -3470,7 +3836,13 @@ static s32 e1000_read_flash_byte_ich8lan(struct e1000_hw *hw, u32 offset, s32 ret_val; u16 word = 0; - ret_val = e1000_read_flash_data_ich8lan(hw, offset, 1, &word); + /* In SPT, only 32 bits access is supported, + * so this function should not be called. + */ + if (hw->mac.type == e1000_pch_spt) + return -E1000_ERR_NVM; + else + ret_val = e1000_read_flash_data_ich8lan(hw, offset, 1, &word); if (ret_val) return ret_val; @@ -3556,6 +3928,83 @@ static s32 e1000_read_flash_data_ich8lan(struct e1000_hw *hw, u32 offset, return ret_val; } +/** + * e1000_read_flash_data32_ich8lan - Read dword from NVM + * @hw: pointer to the HW structure + * @offset: The offset (in bytes) of the dword to read. + * @data: Pointer to the dword to store the value read. + * + * Reads a byte or word from the NVM using the flash access registers. + **/ +static s32 e1000_read_flash_data32_ich8lan(struct e1000_hw *hw, u32 offset, + u32 *data) +{ + union ich8_hws_flash_status hsfsts; + union ich8_hws_flash_ctrl hsflctl; + u32 flash_linear_addr; + s32 ret_val = -E1000_ERR_NVM; + u8 count = 0; + + DEBUGFUNC("e1000_read_flash_data_ich8lan"); + + if (offset > ICH_FLASH_LINEAR_ADDR_MASK || + hw->mac.type != e1000_pch_spt) + return -E1000_ERR_NVM; + flash_linear_addr = ((ICH_FLASH_LINEAR_ADDR_MASK & offset) + + hw->nvm.flash_base_addr); + + do { + usec_delay(1); + /* Steps */ + ret_val = e1000_flash_cycle_init_ich8lan(hw); + if (ret_val != E1000_SUCCESS) + break; + /* In SPT, This register is in Lan memory space, not flash. + * Therefore, only 32 bit access is supported + */ + hsflctl.regval = E1000_READ_FLASH_REG(hw, ICH_FLASH_HSFSTS)>>16; + + /* 0b/1b corresponds to 1 or 2 byte size, respectively. */ + hsflctl.hsf_ctrl.fldbcount = sizeof(u32) - 1; + hsflctl.hsf_ctrl.flcycle = ICH_CYCLE_READ; + /* In SPT, This register is in Lan memory space, not flash. + * Therefore, only 32 bit access is supported + */ + E1000_WRITE_FLASH_REG(hw, ICH_FLASH_HSFSTS, + (u32)hsflctl.regval << 16); + E1000_WRITE_FLASH_REG(hw, ICH_FLASH_FADDR, flash_linear_addr); + + ret_val = e1000_flash_cycle_ich8lan(hw, + ICH_FLASH_READ_COMMAND_TIMEOUT); + + /* Check if FCERR is set to 1, if set to 1, clear it + * and try the whole sequence a few more times, else + * read in (shift in) the Flash Data0, the order is + * least significant byte first msb to lsb + */ + if (ret_val == E1000_SUCCESS) { + *data = E1000_READ_FLASH_REG(hw, ICH_FLASH_FDATA0); + break; + } else { + /* If we've gotten here, then things are probably + * completely hosed, but if the error condition is + * detected, it won't hurt to give it another try... + * ICH_FLASH_CYCLE_REPEAT_COUNT times. + */ + hsfsts.regval = E1000_READ_FLASH_REG16(hw, + ICH_FLASH_HSFSTS); + if (hsfsts.hsf_status.flcerr) { + /* Repeat for some time before giving up. */ + continue; + } else if (!hsfsts.hsf_status.flcdone) { + DEBUGOUT("Timeout error - flash cycle did not complete.\n"); + break; + } + } + } while (count++ < ICH_FLASH_CYCLE_REPEAT_COUNT); + + return ret_val; +} /** * e1000_write_nvm_ich8lan - Write word(s) to the NVM @@ -3594,6 +4043,175 @@ static s32 e1000_write_nvm_ich8lan(struct e1000_hw *hw, u16 offset, u16 words, } /** + * e1000_update_nvm_checksum_spt - Update the checksum for NVM + * @hw: pointer to the HW structure + * + * The NVM checksum is updated by calling the generic update_nvm_checksum, + * which writes the checksum to the shadow ram. The changes in the shadow + * ram are then committed to the EEPROM by processing each bank at a time + * checking for the modified bit and writing only the pending changes. + * After a successful commit, the shadow ram is cleared and is ready for + * future writes. + **/ +static s32 e1000_update_nvm_checksum_spt(struct e1000_hw *hw) +{ + struct e1000_nvm_info *nvm = &hw->nvm; + struct e1000_dev_spec_ich8lan *dev_spec = &hw->dev_spec.ich8lan; + u32 i, act_offset, new_bank_offset, old_bank_offset, bank; + s32 ret_val; + u32 dword = 0; + + DEBUGFUNC("e1000_update_nvm_checksum_spt"); + + ret_val = e1000_update_nvm_checksum_generic(hw); + if (ret_val) + goto out; + + if (nvm->type != e1000_nvm_flash_sw) + goto out; + + nvm->ops.acquire(hw); + + /* We're writing to the opposite bank so if we're on bank 1, + * write to bank 0 etc. We also need to erase the segment that + * is going to be written + */ + ret_val = e1000_valid_nvm_bank_detect_ich8lan(hw, &bank); + if (ret_val != E1000_SUCCESS) { + DEBUGOUT("Could not detect valid bank, assuming bank 0\n"); + bank = 0; + } + + if (bank == 0) { + new_bank_offset = nvm->flash_bank_size; + old_bank_offset = 0; + ret_val = e1000_erase_flash_bank_ich8lan(hw, 1); + if (ret_val) + goto release; + } else { + old_bank_offset = nvm->flash_bank_size; + new_bank_offset = 0; + ret_val = e1000_erase_flash_bank_ich8lan(hw, 0); + if (ret_val) + goto release; + } + for (i = 0; i < E1000_SHADOW_RAM_WORDS; i += 2) { + /* Determine whether to write the value stored + * in the other NVM bank or a modified value stored + * in the shadow RAM + */ + ret_val = e1000_read_flash_dword_ich8lan(hw, + i + old_bank_offset, + &dword); + + if (dev_spec->shadow_ram[i].modified) { + dword &= 0xffff0000; + dword |= (dev_spec->shadow_ram[i].value & 0xffff); + } + if (dev_spec->shadow_ram[i + 1].modified) { + dword &= 0x0000ffff; + dword |= ((dev_spec->shadow_ram[i + 1].value & 0xffff) + << 16); + } + if (ret_val) + break; + + /* If the word is 0x13, then make sure the signature bits + * (15:14) are 11b until the commit has completed. + * This will allow us to write 10b which indicates the + * signature is valid. We want to do this after the write + * has completed so that we don't mark the segment valid + * while the write is still in progress + */ + if (i == E1000_ICH_NVM_SIG_WORD - 1) + dword |= E1000_ICH_NVM_SIG_MASK << 16; + + /* Convert offset to bytes. */ + act_offset = (i + new_bank_offset) << 1; + + usec_delay(100); + + /* Write the data to the new bank. Offset in words*/ + act_offset = i + new_bank_offset; + ret_val = e1000_retry_write_flash_dword_ich8lan(hw, act_offset, + dword); + if (ret_val) + break; + } + + /* Don't bother writing the segment valid bits if sector + * programming failed. + */ + if (ret_val) { + DEBUGOUT("Flash commit failed.\n"); + goto release; + } + + /* Finally validate the new segment by setting bit 15:14 + * to 10b in word 0x13 , this can be done without an + * erase as well since these bits are 11 to start with + * and we need to change bit 14 to 0b + */ + act_offset = new_bank_offset + E1000_ICH_NVM_SIG_WORD; + + /*offset in words but we read dword*/ + --act_offset; + ret_val = e1000_read_flash_dword_ich8lan(hw, act_offset, &dword); + + if (ret_val) + goto release; + + dword &= 0xBFFFFFFF; + ret_val = e1000_retry_write_flash_dword_ich8lan(hw, act_offset, dword); + + if (ret_val) + goto release; + + /* And invalidate the previously valid segment by setting + * its signature word (0x13) high_byte to 0b. This can be + * done without an erase because flash erase sets all bits + * to 1's. We can write 1's to 0's without an erase + */ + act_offset = (old_bank_offset + E1000_ICH_NVM_SIG_WORD) * 2 + 1; + + /* offset in words but we read dword*/ + act_offset = old_bank_offset + E1000_ICH_NVM_SIG_WORD - 1; + ret_val = e1000_read_flash_dword_ich8lan(hw, act_offset, &dword); + + if (ret_val) + goto release; + + dword &= 0x00FFFFFF; + ret_val = e1000_retry_write_flash_dword_ich8lan(hw, act_offset, dword); + + if (ret_val) + goto release; + + /* Great! Everything worked, we can now clear the cached entries. */ + for (i = 0; i < E1000_SHADOW_RAM_WORDS; i++) { + dev_spec->shadow_ram[i].modified = FALSE; + dev_spec->shadow_ram[i].value = 0xFFFF; + } + +release: + nvm->ops.release(hw); + + /* Reload the EEPROM, or else modifications will not appear + * until after the next adapter reset. + */ + if (!ret_val) { + nvm->ops.reload(hw); + msec_delay(10); + } + +out: + if (ret_val) + DEBUGOUT1("NVM update error: %d\n", ret_val); + + return ret_val; +} + +/** * e1000_update_nvm_checksum_ich8lan - Update the checksum for NVM * @hw: pointer to the HW structure * @@ -3770,6 +4388,7 @@ static s32 e1000_validate_nvm_checksum_ich8lan(struct e1000_hw *hw) */ switch (hw->mac.type) { case e1000_pch_lpt: + case e1000_pch_spt: word = NVM_COMPAT; valid_csum_mask = NVM_COMPAT_VALID_CSUM; break; @@ -3817,8 +4436,13 @@ static s32 e1000_write_flash_data_ich8lan(struct e1000_hw *hw, u32 offset, DEBUGFUNC("e1000_write_ich8_data"); - if (size < 1 || size > 2 || offset > ICH_FLASH_LINEAR_ADDR_MASK) - return -E1000_ERR_NVM; + if (hw->mac.type == e1000_pch_spt) { + if (size != 4 || offset > ICH_FLASH_LINEAR_ADDR_MASK) + return -E1000_ERR_NVM; + } else { + if (size < 1 || size > 2 || offset > ICH_FLASH_LINEAR_ADDR_MASK) + return -E1000_ERR_NVM; + } flash_linear_addr = ((ICH_FLASH_LINEAR_ADDR_MASK & offset) + hw->nvm.flash_base_addr); @@ -3829,12 +4453,29 @@ static s32 e1000_write_flash_data_ich8lan(struct e1000_hw *hw, u32 offset, ret_val = e1000_flash_cycle_init_ich8lan(hw); if (ret_val != E1000_SUCCESS) break; - hsflctl.regval = E1000_READ_FLASH_REG16(hw, ICH_FLASH_HSFCTL); + /* In SPT, This register is in Lan memory space, not + * flash. Therefore, only 32 bit access is supported + */ + if (hw->mac.type == e1000_pch_spt) + hsflctl.regval = + E1000_READ_FLASH_REG(hw, ICH_FLASH_HSFSTS)>>16; + else + hsflctl.regval = + E1000_READ_FLASH_REG16(hw, ICH_FLASH_HSFCTL); /* 0b/1b corresponds to 1 or 2 byte size, respectively. */ hsflctl.hsf_ctrl.fldbcount = size - 1; hsflctl.hsf_ctrl.flcycle = ICH_CYCLE_WRITE; - E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFCTL, hsflctl.regval); + /* In SPT, This register is in Lan memory space, + * not flash. Therefore, only 32 bit access is + * supported + */ + if (hw->mac.type == e1000_pch_spt) + E1000_WRITE_FLASH_REG(hw, ICH_FLASH_HSFSTS, + hsflctl.regval << 16); + else + E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFCTL, + hsflctl.regval); E1000_WRITE_FLASH_REG(hw, ICH_FLASH_FADDR, flash_linear_addr); @@ -3872,6 +4513,94 @@ static s32 e1000_write_flash_data_ich8lan(struct e1000_hw *hw, u32 offset, return ret_val; } +/** +* e1000_write_flash_data32_ich8lan - Writes 4 bytes to the NVM +* @hw: pointer to the HW structure +* @offset: The offset (in bytes) of the dwords to read. +* @data: The 4 bytes to write to the NVM. +* +* Writes one/two/four bytes to the NVM using the flash access registers. +**/ +static s32 e1000_write_flash_data32_ich8lan(struct e1000_hw *hw, u32 offset, + u32 data) +{ + union ich8_hws_flash_status hsfsts; + union ich8_hws_flash_ctrl hsflctl; + u32 flash_linear_addr; + s32 ret_val; + u8 count = 0; + + DEBUGFUNC("e1000_write_flash_data32_ich8lan"); + + if (hw->mac.type == e1000_pch_spt) { + if (offset > ICH_FLASH_LINEAR_ADDR_MASK) + return -E1000_ERR_NVM; + } + flash_linear_addr = ((ICH_FLASH_LINEAR_ADDR_MASK & offset) + + hw->nvm.flash_base_addr); + do { + usec_delay(1); + /* Steps */ + ret_val = e1000_flash_cycle_init_ich8lan(hw); + if (ret_val != E1000_SUCCESS) + break; + + /* In SPT, This register is in Lan memory space, not + * flash. Therefore, only 32 bit access is supported + */ + if (hw->mac.type == e1000_pch_spt) + hsflctl.regval = E1000_READ_FLASH_REG(hw, + ICH_FLASH_HSFSTS) + >> 16; + else + hsflctl.regval = E1000_READ_FLASH_REG16(hw, + ICH_FLASH_HSFCTL); + + hsflctl.hsf_ctrl.fldbcount = sizeof(u32) - 1; + hsflctl.hsf_ctrl.flcycle = ICH_CYCLE_WRITE; + + /* In SPT, This register is in Lan memory space, + * not flash. Therefore, only 32 bit access is + * supported + */ + if (hw->mac.type == e1000_pch_spt) + E1000_WRITE_FLASH_REG(hw, ICH_FLASH_HSFSTS, + hsflctl.regval << 16); + else + E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFCTL, + hsflctl.regval); + + E1000_WRITE_FLASH_REG(hw, ICH_FLASH_FADDR, flash_linear_addr); + + E1000_WRITE_FLASH_REG(hw, ICH_FLASH_FDATA0, data); + + /* check if FCERR is set to 1 , if set to 1, clear it + * and try the whole sequence a few more times else done + */ + ret_val = e1000_flash_cycle_ich8lan(hw, + ICH_FLASH_WRITE_COMMAND_TIMEOUT); + + if (ret_val == E1000_SUCCESS) + break; + + /* If we're here, then things are most likely + * completely hosed, but if the error condition + * is detected, it won't hurt to give it another + * try...ICH_FLASH_CYCLE_REPEAT_COUNT times. + */ + hsfsts.regval = E1000_READ_FLASH_REG16(hw, ICH_FLASH_HSFSTS); + + if (hsfsts.hsf_status.flcerr) + /* Repeat for some time before giving up. */ + continue; + if (!hsfsts.hsf_status.flcdone) { + DEBUGOUT("Timeout error - flash cycle did not complete.\n"); + break; + } + } while (count++ < ICH_FLASH_CYCLE_REPEAT_COUNT); + + return ret_val; +} /** * e1000_write_flash_byte_ich8lan - Write a single byte to NVM @@ -3891,7 +4620,42 @@ static s32 e1000_write_flash_byte_ich8lan(struct e1000_hw *hw, u32 offset, return e1000_write_flash_data_ich8lan(hw, offset, 1, word); } +/** +* e1000_retry_write_flash_dword_ich8lan - Writes a dword to NVM +* @hw: pointer to the HW structure +* @offset: The offset of the word to write. +* @dword: The dword to write to the NVM. +* +* Writes a single dword to the NVM using the flash access registers. +* Goes through a retry algorithm before giving up. +**/ +static s32 e1000_retry_write_flash_dword_ich8lan(struct e1000_hw *hw, + u32 offset, u32 dword) +{ + s32 ret_val; + u16 program_retries; + + DEBUGFUNC("e1000_retry_write_flash_dword_ich8lan"); + + /* Must convert word offset into bytes. */ + offset <<= 1; + + ret_val = e1000_write_flash_data32_ich8lan(hw, offset, dword); + + if (!ret_val) + return ret_val; + for (program_retries = 0; program_retries < 100; program_retries++) { + DEBUGOUT2("Retrying Byte %8.8X at offset %u\n", dword, offset); + usec_delay(100); + ret_val = e1000_write_flash_data32_ich8lan(hw, offset, dword); + if (ret_val == E1000_SUCCESS) + break; + } + if (program_retries == 100) + return -E1000_ERR_NVM; + return E1000_SUCCESS; +} /** * e1000_retry_write_flash_byte_ich8lan - Writes a single byte to NVM @@ -4001,12 +4765,22 @@ static s32 e1000_erase_flash_bank_ich8lan(struct e1000_hw *hw, u32 bank) /* Write a value 11 (block Erase) in Flash * Cycle field in hw flash control */ - hsflctl.regval = - E1000_READ_FLASH_REG16(hw, ICH_FLASH_HSFCTL); + if (hw->mac.type == e1000_pch_spt) + hsflctl.regval = + E1000_READ_FLASH_REG(hw, + ICH_FLASH_HSFSTS)>>16; + else + hsflctl.regval = + E1000_READ_FLASH_REG16(hw, + ICH_FLASH_HSFCTL); hsflctl.hsf_ctrl.flcycle = ICH_CYCLE_ERASE; - E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFCTL, - hsflctl.regval); + if (hw->mac.type == e1000_pch_spt) + E1000_WRITE_FLASH_REG(hw, ICH_FLASH_HSFSTS, + hsflctl.regval << 16); + else + E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFCTL, + hsflctl.regval); /* Write the last 24 bits of an index within the * block into Flash Linear address field in Flash @@ -4143,7 +4917,7 @@ static s32 e1000_id_led_init_pchlan(struct e1000_hw *hw) * @hw: pointer to the HW structure * * ICH8 use the PCI Express bus, but does not contain a PCI Express Capability - * register, so the the bus width is hard coded. + * register, so the bus width is hard coded. **/ static s32 e1000_get_bus_info_ich8lan(struct e1000_hw *hw) { @@ -4439,7 +5213,8 @@ static void e1000_initialize_hw_bits_ich8lan(struct e1000_hw *hw) E1000_WRITE_REG(hw, E1000_RFCTL, reg); /* Enable ECC on Lynxpoint */ - if (hw->mac.type == e1000_pch_lpt) { + if ((hw->mac.type == e1000_pch_lpt) || + (hw->mac.type == e1000_pch_spt)) { reg = E1000_READ_REG(hw, E1000_PBECCSTS); reg |= E1000_PBECCSTS_ECC_ENABLE; E1000_WRITE_REG(hw, E1000_PBECCSTS, reg); @@ -4871,7 +5646,8 @@ void e1000_suspend_workarounds_ich8lan(struct e1000_hw *hw) if ((device_id == E1000_DEV_ID_PCH_LPTLP_I218_LM) || (device_id == E1000_DEV_ID_PCH_LPTLP_I218_V) || (device_id == E1000_DEV_ID_PCH_I218_LM3) || - (device_id == E1000_DEV_ID_PCH_I218_V3)) { + (device_id == E1000_DEV_ID_PCH_I218_V3) || + (hw->mac.type == e1000_pch_spt)) { u32 fextnvm6 = E1000_READ_REG(hw, E1000_FEXTNVM6); E1000_WRITE_REG(hw, E1000_FEXTNVM6, @@ -4987,19 +5763,18 @@ out: * the PHY. * On i217, setup Intel Rapid Start Technology. **/ -void e1000_resume_workarounds_pchlan(struct e1000_hw *hw) +u32 e1000_resume_workarounds_pchlan(struct e1000_hw *hw) { s32 ret_val; DEBUGFUNC("e1000_resume_workarounds_pchlan"); - if (hw->mac.type < e1000_pch2lan) - return; + return E1000_SUCCESS; ret_val = e1000_init_phy_workarounds_pchlan(hw); if (ret_val) { DEBUGOUT1("Failed to init PHY flow ret_val=%d\n", ret_val); - return; + return ret_val; } /* For i217 Intel Rapid Start Technology support when the system @@ -5013,7 +5788,7 @@ void e1000_resume_workarounds_pchlan(struct e1000_hw *hw) ret_val = hw->phy.ops.acquire(hw); if (ret_val) { DEBUGOUT("Failed to setup iRST\n"); - return; + return ret_val; } /* Clear Auto Enable LPI after link up */ @@ -5047,7 +5822,9 @@ release: if (ret_val) DEBUGOUT1("Error %d in resume workarounds\n", ret_val); hw->phy.ops.release(hw); + return ret_val; } + return E1000_SUCCESS; } /** diff --git a/freebsd/sys/dev/e1000/e1000_ich8lan.h b/freebsd/sys/dev/e1000/e1000_ich8lan.h index 999e856b..6d812911 100644 --- a/freebsd/sys/dev/e1000/e1000_ich8lan.h +++ b/freebsd/sys/dev/e1000/e1000_ich8lan.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -107,9 +107,23 @@ #define E1000_FEXTNVM6_REQ_PLL_CLK 0x00000100 #define E1000_FEXTNVM6_ENABLE_K1_ENTRY_CONDITION 0x00000200 - +#define E1000_FEXTNVM6_K1_OFF_ENABLE 0x80000000 +/* bit for disabling packet buffer read */ +#define E1000_FEXTNVM7_DISABLE_PB_READ 0x00040000 +#define E1000_FEXTNVM7_SIDE_CLK_UNGATE 0x00000004 #define E1000_FEXTNVM7_DISABLE_SMB_PERST 0x00000020 - +#define E1000_FEXTNVM9_IOSFSB_CLKGATE_DIS 0x00000800 +#define E1000_FEXTNVM9_IOSFSB_CLKREQ_DIS 0x00001000 +#define E1000_FEXTNVM11_DISABLE_PB_READ 0x00000200 +#define E1000_FEXTNVM11_DISABLE_MULR_FIX 0x00002000 + +/* bit24: RXDCTL thresholds granularity: 0 - cache lines, 1 - descriptors */ +#define E1000_RXDCTL_THRESH_UNIT_DESC 0x01000000 + +#define NVM_SIZE_MULTIPLIER 4096 /*multiplier for NVMS field*/ +#define E1000_FLASH_BASE_ADDR 0xE000 /*offset of NVM access regs*/ +#define E1000_CTRL_EXT_NVMVS 0x3 /*NVM valid sector */ +#define E1000_TARC0_CB_MULTIQ_3_REQ (1 << 28 | 1 << 29) #define PCIE_ICH8_SNOOP_ALL PCIE_NO_SNOOP_ALL #define E1000_ICH_RAR_ENTRIES 7 @@ -171,6 +185,8 @@ #define E1000_NVM_K1_CONFIG 0x1B /* NVM K1 Config Word */ #define E1000_NVM_K1_ENABLE 0x1 /* NVM Enable K1 bit */ +#define K1_ENTRY_LATENCY 0 +#define K1_MIN_TIME 1 /* SMBus Control Phy Register */ #define CV_SMB_CTRL PHY_REG(769, 23) @@ -184,6 +200,10 @@ #define I218_ULP_CONFIG1_INBAND_EXIT 0x0020 /* Inband on ULP exit */ #define I218_ULP_CONFIG1_WOL_HOST 0x0040 /* WoL Host on ULP exit */ #define I218_ULP_CONFIG1_RESET_TO_SMBUS 0x0100 /* Reset to SMBus mode */ +/* enable ULP even if when phy powered down via lanphypc */ +#define I218_ULP_CONFIG1_EN_ULP_LANPHYPC 0x0400 +/* disable clear of sticky ULP on PERST */ +#define I218_ULP_CONFIG1_DIS_CLR_STICKY_ON_PERST 0x0800 #define I218_ULP_CONFIG1_DISABLE_SMB_PERST 0x1000 /* Disable on PERST# */ /* SMBus Address Phy Register */ @@ -219,9 +239,12 @@ /* PHY Power Management Control */ #define HV_PM_CTRL PHY_REG(770, 17) -#define HV_PM_CTRL_PLL_STOP_IN_K1_GIGA 0x100 +#define HV_PM_CTRL_K1_CLK_REQ 0x200 #define HV_PM_CTRL_K1_ENABLE 0x4000 +#define I217_PLL_CLOCK_GATE_REG PHY_REG(772, 28) +#define I217_PLL_CLOCK_GATE_MASK 0x07FF + #define SW_FLAG_TIMEOUT 1000 /* SW Semaphore flag timeout in ms */ /* Inband Control */ @@ -307,7 +330,7 @@ void e1000_set_kmrn_lock_loss_workaround_ich8lan(struct e1000_hw *hw, void e1000_igp3_phy_powerdown_workaround_ich8lan(struct e1000_hw *hw); void e1000_gig_downshift_workaround_ich8lan(struct e1000_hw *hw); void e1000_suspend_workarounds_ich8lan(struct e1000_hw *hw); -void e1000_resume_workarounds_pchlan(struct e1000_hw *hw); +u32 e1000_resume_workarounds_pchlan(struct e1000_hw *hw); s32 e1000_configure_k1_ich8lan(struct e1000_hw *hw, bool k1_enable); void e1000_copy_rx_addrs_to_phy_ich8lan(struct e1000_hw *hw); s32 e1000_lv_jumbo_workaround_ich8lan(struct e1000_hw *hw, bool enable); diff --git a/freebsd/sys/dev/e1000/e1000_mac.c b/freebsd/sys/dev/e1000/e1000_mac.c index 3967a25a..2af37399 100644 --- a/freebsd/sys/dev/e1000/e1000_mac.c +++ b/freebsd/sys/dev/e1000/e1000_mac.c @@ -2,7 +2,7 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/freebsd/sys/dev/e1000/e1000_mac.h b/freebsd/sys/dev/e1000/e1000_mac.h index 2c1bfe32..ef9789bb 100644 --- a/freebsd/sys/dev/e1000/e1000_mac.h +++ b/freebsd/sys/dev/e1000/e1000_mac.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -36,9 +36,7 @@ #define _E1000_MAC_H_ void e1000_init_mac_ops_generic(struct e1000_hw *hw); -#ifndef E1000_REMOVED #define E1000_REMOVED(a) (0) -#endif /* E1000_REMOVED */ void e1000_null_mac_generic(struct e1000_hw *hw); s32 e1000_null_ops_generic(struct e1000_hw *hw); s32 e1000_null_link_info(struct e1000_hw *hw, u16 *s, u16 *d); diff --git a/freebsd/sys/dev/e1000/e1000_manage.c b/freebsd/sys/dev/e1000/e1000_manage.c index 0648ac9d..82456912 100644 --- a/freebsd/sys/dev/e1000/e1000_manage.c +++ b/freebsd/sys/dev/e1000/e1000_manage.c @@ -2,7 +2,7 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -35,7 +35,6 @@ /*$FreeBSD$*/ #include "e1000_api.h" - /** * e1000_calculate_checksum - Calculate checksum for buffer * @buffer: pointer to EEPROM diff --git a/freebsd/sys/dev/e1000/e1000_manage.h b/freebsd/sys/dev/e1000/e1000_manage.h index 51f17671..303e99e3 100644 --- a/freebsd/sys/dev/e1000/e1000_manage.h +++ b/freebsd/sys/dev/e1000/e1000_manage.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2012, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/freebsd/sys/dev/e1000/e1000_mbx.c b/freebsd/sys/dev/e1000/e1000_mbx.c index 1b5bb70d..0c6bb2c2 100644 --- a/freebsd/sys/dev/e1000/e1000_mbx.c +++ b/freebsd/sys/dev/e1000/e1000_mbx.c @@ -2,7 +2,7 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -69,7 +69,7 @@ static s32 e1000_null_mbx_transact(struct e1000_hw E1000_UNUSEDARG *hw, * @size: Length of buffer * @mbx_id: id of mailbox to read * - * returns SUCCESS if it successfuly read message from buffer + * returns SUCCESS if it successfully read message from buffer **/ s32 e1000_read_mbx(struct e1000_hw *hw, u32 *msg, u16 size, u16 mbx_id) { @@ -428,15 +428,21 @@ static s32 e1000_check_for_rst_vf(struct e1000_hw *hw, static s32 e1000_obtain_mbx_lock_vf(struct e1000_hw *hw) { s32 ret_val = -E1000_ERR_MBX; + int count = 10; DEBUGFUNC("e1000_obtain_mbx_lock_vf"); - /* Take ownership of the buffer */ - E1000_WRITE_REG(hw, E1000_V2PMAILBOX(0), E1000_V2PMAILBOX_VFU); + do { + /* Take ownership of the buffer */ + E1000_WRITE_REG(hw, E1000_V2PMAILBOX(0), E1000_V2PMAILBOX_VFU); - /* reserve mailbox for vf use */ - if (e1000_read_v2p_mailbox(hw) & E1000_V2PMAILBOX_VFU) - ret_val = E1000_SUCCESS; + /* reserve mailbox for vf use */ + if (e1000_read_v2p_mailbox(hw) & E1000_V2PMAILBOX_VFU) { + ret_val = E1000_SUCCESS; + break; + } + usec_delay(1000); + } while (count-- > 0); return ret_val; } @@ -489,7 +495,7 @@ out_no_write: * @size: Length of buffer * @mbx_id: id of mailbox to read * - * returns SUCCESS if it successfuly read message from buffer + * returns SUCCESS if it successfully read message from buffer **/ static s32 e1000_read_mbx_vf(struct e1000_hw *hw, u32 *msg, u16 size, u16 E1000_UNUSEDARG mbx_id) @@ -641,18 +647,26 @@ static s32 e1000_obtain_mbx_lock_pf(struct e1000_hw *hw, u16 vf_number) { s32 ret_val = -E1000_ERR_MBX; u32 p2v_mailbox; + int count = 10; DEBUGFUNC("e1000_obtain_mbx_lock_pf"); - /* Take ownership of the buffer */ - E1000_WRITE_REG(hw, E1000_P2VMAILBOX(vf_number), E1000_P2VMAILBOX_PFU); + do { + /* Take ownership of the buffer */ + E1000_WRITE_REG(hw, E1000_P2VMAILBOX(vf_number), + E1000_P2VMAILBOX_PFU); - /* reserve mailbox for vf use */ - p2v_mailbox = E1000_READ_REG(hw, E1000_P2VMAILBOX(vf_number)); - if (p2v_mailbox & E1000_P2VMAILBOX_PFU) - ret_val = E1000_SUCCESS; + /* reserve mailbox for pf use */ + p2v_mailbox = E1000_READ_REG(hw, E1000_P2VMAILBOX(vf_number)); + if (p2v_mailbox & E1000_P2VMAILBOX_PFU) { + ret_val = E1000_SUCCESS; + break; + } + usec_delay(1000); + } while (count-- > 0); return ret_val; + } /** diff --git a/freebsd/sys/dev/e1000/e1000_mbx.h b/freebsd/sys/dev/e1000/e1000_mbx.h index d2aea5c4..fadd8494 100644 --- a/freebsd/sys/dev/e1000/e1000_mbx.h +++ b/freebsd/sys/dev/e1000/e1000_mbx.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/freebsd/sys/dev/e1000/e1000_nvm.c b/freebsd/sys/dev/e1000/e1000_nvm.c index ad0c9544..a1e881ad 100644 --- a/freebsd/sys/dev/e1000/e1000_nvm.c +++ b/freebsd/sys/dev/e1000/e1000_nvm.c @@ -2,7 +2,7 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/freebsd/sys/dev/e1000/e1000_nvm.h b/freebsd/sys/dev/e1000/e1000_nvm.h index 34077b24..64a4083e 100644 --- a/freebsd/sys/dev/e1000/e1000_nvm.h +++ b/freebsd/sys/dev/e1000/e1000_nvm.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2013, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -35,12 +35,10 @@ #ifndef _E1000_NVM_H_ #define _E1000_NVM_H_ -#if !defined(NO_READ_PBA_RAW) || !defined(NO_WRITE_PBA_RAW) struct e1000_pba { u16 word[2]; u16 *pba_block; }; -#endif void e1000_init_nvm_ops_generic(struct e1000_hw *hw); diff --git a/freebsd/sys/dev/e1000/e1000_osdep.c b/freebsd/sys/dev/e1000/e1000_osdep.c index 7eef489c..45aae330 100644 --- a/freebsd/sys/dev/e1000/e1000_osdep.c +++ b/freebsd/sys/dev/e1000/e1000_osdep.c @@ -2,7 +2,7 @@ /****************************************************************************** - Copyright (c) 2001-2010, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/freebsd/sys/dev/e1000/e1000_osdep.h b/freebsd/sys/dev/e1000/e1000_osdep.h index b58a85b4..4ef3ce76 100644 --- a/freebsd/sys/dev/e1000/e1000_osdep.h +++ b/freebsd/sys/dev/e1000/e1000_osdep.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -60,24 +60,24 @@ #define ASSERT(x) if(!(x)) panic("EM: x") #define usec_delay(x) DELAY(x) -#define usec_delay_irq(x) DELAY(x) +#define usec_delay_irq(x) usec_delay(x) #define msec_delay(x) DELAY(1000*(x)) #define msec_delay_irq(x) DELAY(1000*(x)) -#define DEBUGFUNC(F) DEBUGOUT(F); -#define DEBUGOUT(S) do {} while (0) -#define DEBUGOUT1(S,A) do {} while (0) -#define DEBUGOUT2(S,A,B) do {} while (0) -#define DEBUGOUT3(S,A,B,C) do {} while (0) -#define DEBUGOUT7(S,A,B,C,D,E,F,G) do {} while (0) +/* Enable/disable debugging statements in shared code */ +#define DBG 0 + +#define DEBUGOUT(...) \ + do { if (DBG) printf(__VA_ARGS__); } while (0) +#define DEBUGOUT1(...) DEBUGOUT(__VA_ARGS__) +#define DEBUGOUT2(...) DEBUGOUT(__VA_ARGS__) +#define DEBUGOUT3(...) DEBUGOUT(__VA_ARGS__) +#define DEBUGOUT7(...) DEBUGOUT(__VA_ARGS__) +#define DEBUGFUNC(F) DEBUGOUT(F "\n") #define STATIC static #define FALSE 0 #define TRUE 1 -#ifndef __bool_true_false_are_defined -#define false FALSE -#define true TRUE -#endif #define CMD_MEM_WRT_INVALIDATE 0x0010 /* BIT_4 */ #define PCI_COMMAND_REGISTER PCIR_COMMAND @@ -99,9 +99,6 @@ typedef int64_t s64; typedef int32_t s32; typedef int16_t s16; typedef int8_t s8; -#ifndef __bool_true_false_are_defined -typedef boolean_t bool; -#endif #define __le16 u16 #define __le32 u32 @@ -137,7 +134,7 @@ struct e1000_osdep bus_space_handle_t io_bus_space_handle; bus_space_tag_t flash_bus_space_tag; bus_space_handle_t flash_bus_space_handle; - struct device *dev; + device_t dev; }; #define E1000_REGISTER(hw, reg) (((hw)->mac.type >= e1000_82543) \ diff --git a/freebsd/sys/dev/e1000/e1000_phy.c b/freebsd/sys/dev/e1000/e1000_phy.c index cb92973b..87753e19 100644 --- a/freebsd/sys/dev/e1000/e1000_phy.c +++ b/freebsd/sys/dev/e1000/e1000_phy.c @@ -2,7 +2,7 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -1829,9 +1829,9 @@ s32 e1000_phy_force_speed_duplex_m88(struct e1000_hw *hw) phy_data); if (ret_val) return ret_val; - } - DEBUGOUT1("M88E1000 PSCR: %X\n", phy_data); + DEBUGOUT1("M88E1000 PSCR: %X\n", phy_data); + } ret_val = phy->ops.read_reg(hw, PHY_CONTROL, &phy_data); if (ret_val) @@ -3124,7 +3124,7 @@ s32 e1000_write_phy_reg_bm(struct e1000_hw *hw, u32 offset, u16 data) /* Page 800 works differently than the rest so it has its own func */ if (page == BM_WUC_PAGE) { ret_val = e1000_access_phy_wakeup_reg_bm(hw, offset, &data, - FALSE, FALSE); + FALSE, false); goto release; } @@ -3288,7 +3288,7 @@ s32 e1000_write_phy_reg_bm2(struct e1000_hw *hw, u32 offset, u16 data) /* Page 800 works differently than the rest so it has its own func */ if (page == BM_WUC_PAGE) { ret_val = e1000_access_phy_wakeup_reg_bm(hw, offset, &data, - FALSE, FALSE); + FALSE, false); goto release; } @@ -3431,12 +3431,11 @@ static s32 e1000_access_phy_wakeup_reg_bm(struct e1000_hw *hw, u32 offset, u16 *data, bool read, bool page_set) { s32 ret_val; - u16 reg, page; + u16 reg = BM_PHY_REG_NUM(offset); + u16 page = BM_PHY_REG_PAGE(offset); u16 phy_reg = 0; DEBUGFUNC("e1000_access_phy_wakeup_reg_bm"); - reg = BM_PHY_REG_NUM(offset); - page = BM_PHY_REG_PAGE(offset); /* Gig must be disabled for MDIO accesses to Host Wakeup reg page */ if ((hw->mac.type == e1000_pchlan) && @@ -3546,7 +3545,6 @@ static s32 __e1000_read_phy_reg_hv(struct e1000_hw *hw, u32 offset, u16 *data, if (ret_val) return ret_val; } - /* Page 800 works differently than the rest so it has its own func */ if (page == BM_WUC_PAGE) { ret_val = e1000_access_phy_wakeup_reg_bm(hw, offset, data, @@ -3600,7 +3598,7 @@ out: **/ s32 e1000_read_phy_reg_hv(struct e1000_hw *hw, u32 offset, u16 *data) { - return __e1000_read_phy_reg_hv(hw, offset, data, FALSE, FALSE); + return __e1000_read_phy_reg_hv(hw, offset, data, FALSE, false); } /** @@ -3656,7 +3654,6 @@ static s32 __e1000_write_phy_reg_hv(struct e1000_hw *hw, u32 offset, u16 data, if (ret_val) return ret_val; } - /* Page 800 works differently than the rest so it has its own func */ if (page == BM_WUC_PAGE) { ret_val = e1000_access_phy_wakeup_reg_bm(hw, offset, &data, @@ -3726,7 +3723,7 @@ out: **/ s32 e1000_write_phy_reg_hv(struct e1000_hw *hw, u32 offset, u16 data) { - return __e1000_write_phy_reg_hv(hw, offset, data, FALSE, FALSE); + return __e1000_write_phy_reg_hv(hw, offset, data, FALSE, false); } /** @@ -4153,10 +4150,10 @@ s32 e1000_read_phy_reg_mphy(struct e1000_hw *hw, u32 address, u32 *data) /* Disable access to mPHY if it was originally disabled */ if (locked) ready = e1000_is_mphy_ready(hw); - if (!ready) - return -E1000_ERR_PHY; - E1000_WRITE_REG(hw, E1000_MPHY_ADDR_CTRL, - E1000_MPHY_DIS_ACCESS); + if (!ready) + return -E1000_ERR_PHY; + E1000_WRITE_REG(hw, E1000_MPHY_ADDR_CTRL, + E1000_MPHY_DIS_ACCESS); return E1000_SUCCESS; } @@ -4218,10 +4215,10 @@ s32 e1000_write_phy_reg_mphy(struct e1000_hw *hw, u32 address, u32 data, /* Disable access to mPHY if it was originally disabled */ if (locked) ready = e1000_is_mphy_ready(hw); - if (!ready) - return -E1000_ERR_PHY; - E1000_WRITE_REG(hw, E1000_MPHY_ADDR_CTRL, - E1000_MPHY_DIS_ACCESS); + if (!ready) + return -E1000_ERR_PHY; + E1000_WRITE_REG(hw, E1000_MPHY_ADDR_CTRL, + E1000_MPHY_DIS_ACCESS); return E1000_SUCCESS; } diff --git a/freebsd/sys/dev/e1000/e1000_phy.h b/freebsd/sys/dev/e1000/e1000_phy.h index 0e5b2e6a..d3d563f7 100644 --- a/freebsd/sys/dev/e1000/e1000_phy.h +++ b/freebsd/sys/dev/e1000/e1000_phy.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/freebsd/sys/dev/e1000/e1000_regs.h b/freebsd/sys/dev/e1000/e1000_regs.h index 5c2e3f78..37d70172 100644 --- a/freebsd/sys/dev/e1000/e1000_regs.h +++ b/freebsd/sys/dev/e1000/e1000_regs.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -65,6 +65,9 @@ #define E1000_FEXTNVM4 0x00024 /* Future Extended NVM 4 - RW */ #define E1000_FEXTNVM6 0x00010 /* Future Extended NVM 6 - RW */ #define E1000_FEXTNVM7 0x000E4 /* Future Extended NVM 7 - RW */ +#define E1000_FEXTNVM9 0x5BB4 /* Future Extended NVM 9 - RW */ +#define E1000_FEXTNVM11 0x5BBC /* Future Extended NVM 11 - RW */ +#define E1000_PCIEANACFG 0x00F18 /* PCIE Analog Config */ #define E1000_FCT 0x00030 /* Flow Control Type - RW */ #define E1000_CONNSW 0x00034 /* Copper/Fiber switch control - RW */ #define E1000_VET 0x00038 /* VLAN Ether Type - RW */ @@ -107,7 +110,9 @@ #define E1000_PBA 0x01000 /* Packet Buffer Allocation - RW */ #define E1000_PBS 0x01008 /* Packet Buffer Size */ #define E1000_PBECCSTS 0x0100C /* Packet Buffer ECC Status - RW */ +#define E1000_IOSFPC 0x00F28 /* TX corrupted data */ #define E1000_EEMNGCTL 0x01010 /* MNG EEprom Control */ +#define E1000_EEMNGCTL_I210 0x01010 /* i210 MNG EEprom Mode Control */ #define E1000_EEARBC 0x01024 /* EEPROM Auto Read Bus Control */ #define E1000_EEARBC_I210 0x12024 /* EEPROM Auto Read Bus Control */ #define E1000_FLASHT 0x01028 /* FLASH Timer Register */ @@ -202,7 +207,7 @@ /* Queues fetch arbitration priority control register */ #define E1000_I210_TQAVARBCTRL 0x3574 /* Queues priority masks where _n and _p can be 0-3. */ -#define E1000_TQAVARBCTRL_QUEUE_PRI(_n, _p) ((_p) << (2 * _n)) +#define E1000_TQAVARBCTRL_QUEUE_PRI(_n, _p) ((_p) << (2 * (_n))) /* QAV Tx mode control registers where _n can be 0 or 1. */ #define E1000_I210_TQAVCC(_n) (0x3004 + 0x40 * (_n)) @@ -215,7 +220,7 @@ #define E1000_PQGPTC(_n) (0x010014 + (0x100 * (_n))) /* Queues packet buffer size masks where _n can be 0-3 and _s 0-63 [kB] */ -#define E1000_I210_TXPBS_SIZE(_n, _s) ((_s) << (6 * _n)) +#define E1000_I210_TXPBS_SIZE(_n, _s) ((_s) << (6 * (_n))) #define E1000_MMDAC 13 /* MMD Access Control */ #define E1000_MMDAAD 14 /* MMD Access Address/Data */ @@ -552,7 +557,7 @@ #define E1000_WVBR 0x03554 /* VM Wrong Behavior - RWS */ #define E1000_RPLOLR 0x05AF0 /* Replication Offload - RW */ #define E1000_UTA 0x0A000 /* Unicast Table Array - RW */ -#define E1000_IOVTCL 0x05BBC /* IOV Control Register */ +#define E1000_IOVCTL 0x05BBC /* IOV Control Register */ #define E1000_VMRCTL 0X05D80 /* Virtual Mirror Rule Control */ #define E1000_VMRVLAN 0x05D90 /* Virtual Mirror Rule VLAN */ #define E1000_VMRVM 0x05DA0 /* Virtual Mirror Rule VM */ @@ -588,6 +593,10 @@ #define E1000_TIMADJL 0x0B60C /* Time sync time adjustment offset Low - RW */ #define E1000_TIMADJH 0x0B610 /* Time sync time adjustment offset High - RW */ #define E1000_TSAUXC 0x0B640 /* Timesync Auxiliary Control register */ +#define E1000_SYSSTMPL 0x0B648 /* HH Timesync system stamp low register */ +#define E1000_SYSSTMPH 0x0B64C /* HH Timesync system stamp hi register */ +#define E1000_PLTSTMPL 0x0B640 /* HH Timesync platform stamp low register */ +#define E1000_PLTSTMPH 0x0B644 /* HH Timesync platform stamp hi register */ #define E1000_SYSTIMR 0x0B6F8 /* System time register Residue */ #define E1000_TSICR 0x0B66C /* Interrupt Cause Register */ #define E1000_TSIM 0x0B674 /* Interrupt Mask Register */ diff --git a/freebsd/sys/dev/e1000/e1000_vf.c b/freebsd/sys/dev/e1000/e1000_vf.c index 17fd7cb0..0e46641e 100644 --- a/freebsd/sys/dev/e1000/e1000_vf.c +++ b/freebsd/sys/dev/e1000/e1000_vf.c @@ -2,7 +2,7 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/freebsd/sys/dev/e1000/e1000_vf.h b/freebsd/sys/dev/e1000/e1000_vf.h index 2a780741..e6f834e7 100644 --- a/freebsd/sys/dev/e1000/e1000_vf.h +++ b/freebsd/sys/dev/e1000/e1000_vf.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/freebsd/sys/dev/e1000/if_em.c b/freebsd/sys/dev/e1000/if_em.c index 05d7fbc2..fa34dd62 100644 --- a/freebsd/sys/dev/e1000/if_em.c +++ b/freebsd/sys/dev/e1000/if_em.c @@ -2,7 +2,7 @@ /****************************************************************************** - Copyright (c) 2001-2014, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -34,6 +34,8 @@ ******************************************************************************/ /*$FreeBSD$*/ +#include <rtems/bsd/local/opt_em.h> +#include <rtems/bsd/local/opt_ddb.h> #include <rtems/bsd/local/opt_inet.h> #include <rtems/bsd/local/opt_inet6.h> @@ -43,6 +45,10 @@ #include <rtems/bsd/sys/param.h> #include <sys/systm.h> +#ifdef DDB +#include <sys/types.h> +#include <ddb/ddb.h> +#endif #if __FreeBSD_version >= 800000 #include <sys/buf_ring.h> #endif @@ -54,6 +60,7 @@ #include <sys/mbuf.h> #include <sys/module.h> #include <sys/rman.h> +#include <sys/smp.h> #include <sys/socket.h> #include <sys/sockio.h> #include <sys/sysctl.h> @@ -65,6 +72,7 @@ #include <net/bpf.h> #include <net/ethernet.h> #include <net/if.h> +#include <net/if_var.h> #include <net/if_arp.h> #include <net/if_dl.h> #include <net/if_media.h> @@ -90,14 +98,9 @@ #include "if_em.h" /********************************************************************* - * Set this to one to display debug statistics - *********************************************************************/ -int em_display_debug_stats = 0; - -/********************************************************************* * Driver version: *********************************************************************/ -char em_driver_version[] = "7.4.2"; +char em_driver_version[] = "7.6.1-k"; /********************************************************************* * PCI Device ID Table @@ -185,6 +188,19 @@ static em_vendor_info_t em_vendor_info_array[] = { 0x8086, E1000_DEV_ID_PCH_I218_V2, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_PCH_I218_LM3, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_PCH_I218_V3, PCI_ANY_ID, PCI_ANY_ID, 0}, + { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, + { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V, PCI_ANY_ID, PCI_ANY_ID, 0}, + { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM2, + PCI_ANY_ID, PCI_ANY_ID, 0}, + { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V2, PCI_ANY_ID, PCI_ANY_ID, 0}, + { 0x8086, E1000_DEV_ID_PCH_LBG_I219_LM3, + PCI_ANY_ID, PCI_ANY_ID, 0}, + { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM4, + PCI_ANY_ID, PCI_ANY_ID, 0}, + { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V4, PCI_ANY_ID, PCI_ANY_ID, 0}, + { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM5, + PCI_ANY_ID, PCI_ANY_ID, 0}, + { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V5, PCI_ANY_ID, PCI_ANY_ID, 0}, /* required last entry */ { 0, 0, 0, 0, 0} }; @@ -207,20 +223,21 @@ static int em_shutdown(device_t); static int em_suspend(device_t); static int em_resume(device_t); #ifdef EM_MULTIQUEUE -static int em_mq_start(struct ifnet *, struct mbuf *); -static int em_mq_start_locked(struct ifnet *, - struct tx_ring *, struct mbuf *); -static void em_qflush(struct ifnet *); +static int em_mq_start(if_t, struct mbuf *); +static int em_mq_start_locked(if_t, + struct tx_ring *); +static void em_qflush(if_t); #else -static void em_start(struct ifnet *); -static void em_start_locked(struct ifnet *, struct tx_ring *); +static void em_start(if_t); +static void em_start_locked(if_t, struct tx_ring *); #endif -static int em_ioctl(struct ifnet *, u_long, caddr_t); +static int em_ioctl(if_t, u_long, caddr_t); +static uint64_t em_get_counter(if_t, ift_counter); static void em_init(void *); static void em_init_locked(struct adapter *); static void em_stop(void *); -static void em_media_status(struct ifnet *, struct ifmediareq *); -static int em_media_change(struct ifnet *); +static void em_media_status(if_t, struct ifmediareq *); +static int em_media_change(if_t); static void em_identify_hardware(struct adapter *); static int em_allocate_pci_resources(struct adapter *); static int em_allocate_legacy(struct adapter *); @@ -231,6 +248,7 @@ static void em_free_pci_resources(struct adapter *); static void em_local_timer(void *); static void em_reset(struct adapter *); static int em_setup_interface(device_t, struct adapter *); +static void em_flush_desc_rings(struct adapter *); static void em_setup_transmit_structures(struct adapter *); static void em_initialize_transmit_unit(struct adapter *); @@ -253,7 +271,9 @@ static bool em_rxeof(struct rx_ring *, int, int *); #ifndef __NO_STRICT_ALIGNMENT static int em_fixup_rx(struct rx_ring *); #endif -static void em_receive_checksum(struct e1000_rx_desc *, struct mbuf *); +static void em_setup_rxdesc(union e1000_rx_desc_extended *, + const struct em_rxbuffer *rxbuf); +static void em_receive_checksum(uint32_t status, struct mbuf *); static void em_transmit_checksum_setup(struct tx_ring *, struct mbuf *, int, struct ip *, u32 *, u32 *); static void em_tso_setup(struct tx_ring *, struct mbuf *, int, struct ip *, @@ -263,8 +283,8 @@ static void em_disable_promisc(struct adapter *); static void em_set_multi(struct adapter *); static void em_update_link_status(struct adapter *); static void em_refresh_mbufs(struct rx_ring *, int); -static void em_register_vlan(void *, struct ifnet *, u16); -static void em_unregister_vlan(void *, struct ifnet *, u16); +static void em_register_vlan(void *, if_t, u16); +static void em_unregister_vlan(void *, if_t, u16); static void em_setup_vlan_hw_support(struct adapter *); static int em_xmit(struct tx_ring *, struct mbuf **); static int em_dma_malloc(struct adapter *, bus_size_t, @@ -299,6 +319,10 @@ static void em_handle_tx(void *context, int pending); static void em_handle_rx(void *context, int pending); static void em_handle_link(void *context, int pending); +#ifdef EM_MULTIQUEUE +static void em_enable_vectors_82574(struct adapter *); +#endif + static void em_set_sysctl_value(struct adapter *, const char *, const char *, int *, int); static int em_set_flowcntl(SYSCTL_HANDLER_ARGS); @@ -333,6 +357,9 @@ devclass_t em_devclass; DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0); MODULE_DEPEND(em, pci, 1, 1, 1); MODULE_DEPEND(em, ether, 1, 1, 1); +#ifdef DEV_NETMAP +MODULE_DEPEND(em, netmap, 1, 1, 1); +#endif /* DEV_NETMAP */ /********************************************************************* * Tunable default values. @@ -350,12 +377,16 @@ MODULE_DEPEND(em, ether, 1, 1, 1); #define CSUM_TSO 0 #endif +#define TSO_WORKAROUND 4 + static SYSCTL_NODE(_hw, OID_AUTO, em, CTLFLAG_RD, 0, "EM driver parameters"); +static int em_disable_crc_stripping = 0; +SYSCTL_INT(_hw_em, OID_AUTO, disable_crc_stripping, CTLFLAG_RDTUN, + &em_disable_crc_stripping, 0, "Disable CRC Stripping"); + static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV); static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR); -TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt); -TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt); SYSCTL_INT(_hw_em, OID_AUTO, tx_int_delay, CTLFLAG_RDTUN, &em_tx_int_delay_dflt, 0, "Default transmit interrupt delay in usecs"); SYSCTL_INT(_hw_em, OID_AUTO, rx_int_delay, CTLFLAG_RDTUN, &em_rx_int_delay_dflt, @@ -363,8 +394,6 @@ SYSCTL_INT(_hw_em, OID_AUTO, rx_int_delay, CTLFLAG_RDTUN, &em_rx_int_delay_dflt, static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV); static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV); -TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt); -TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt); SYSCTL_INT(_hw_em, OID_AUTO, tx_abs_int_delay, CTLFLAG_RDTUN, &em_tx_abs_int_delay_dflt, 0, "Default transmit interrupt delay limit in usecs"); @@ -374,32 +403,39 @@ SYSCTL_INT(_hw_em, OID_AUTO, rx_abs_int_delay, CTLFLAG_RDTUN, static int em_rxd = EM_DEFAULT_RXD; static int em_txd = EM_DEFAULT_TXD; -TUNABLE_INT("hw.em.rxd", &em_rxd); -TUNABLE_INT("hw.em.txd", &em_txd); SYSCTL_INT(_hw_em, OID_AUTO, rxd, CTLFLAG_RDTUN, &em_rxd, 0, "Number of receive descriptors per queue"); SYSCTL_INT(_hw_em, OID_AUTO, txd, CTLFLAG_RDTUN, &em_txd, 0, "Number of transmit descriptors per queue"); static int em_smart_pwr_down = FALSE; -TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down); SYSCTL_INT(_hw_em, OID_AUTO, smart_pwr_down, CTLFLAG_RDTUN, &em_smart_pwr_down, 0, "Set to true to leave smart power down enabled on newer adapters"); /* Controls whether promiscuous also shows bad packets */ static int em_debug_sbp = FALSE; -TUNABLE_INT("hw.em.sbp", &em_debug_sbp); SYSCTL_INT(_hw_em, OID_AUTO, sbp, CTLFLAG_RDTUN, &em_debug_sbp, 0, "Show bad packets in promiscuous mode"); static int em_enable_msix = TRUE; -TUNABLE_INT("hw.em.enable_msix", &em_enable_msix); SYSCTL_INT(_hw_em, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &em_enable_msix, 0, "Enable MSI-X interrupts"); +#ifdef EM_MULTIQUEUE +static int em_num_queues = 1; +SYSCTL_INT(_hw_em, OID_AUTO, num_queues, CTLFLAG_RDTUN, &em_num_queues, 0, + "82574 only: Number of queues to configure, 0 indicates autoconfigure"); +#endif + +/* +** Global variable to store last used CPU when binding queues +** to CPUs in igb_allocate_msix. Starts at CPU_FIRST and increments when a +** queue is bound to a cpu. +*/ +static int em_last_bind_cpu = -1; + /* How many packets rxeof tries to clean at a time */ static int em_rx_process_limit = 100; -TUNABLE_INT("hw.em.rx_process_limit", &em_rx_process_limit); SYSCTL_INT(_hw_em, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, &em_rx_process_limit, 0, "Maximum number of received packets to process " @@ -407,7 +443,6 @@ SYSCTL_INT(_hw_em, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, /* Energy efficient ethernet - default to OFF */ static int eee_setting = 1; -TUNABLE_INT("hw.em.eee_setting", &eee_setting); SYSCTL_INT(_hw_em, OID_AUTO, eee_setting, CTLFLAG_RDTUN, &eee_setting, 0, "Enable Energy Efficient Ethernet"); @@ -431,10 +466,10 @@ static int em_probe(device_t dev) { char adapter_name[60]; - u16 pci_vendor_id = 0; - u16 pci_device_id = 0; - u16 pci_subvendor_id = 0; - u16 pci_subdevice_id = 0; + uint16_t pci_vendor_id = 0; + uint16_t pci_device_id = 0; + uint16_t pci_subvendor_id = 0; + uint16_t pci_subdevice_id = 0; em_vendor_info_t *ent; INIT_DEBUGOUT("em_probe: begin"); @@ -553,14 +588,34 @@ em_attach(device_t dev) adapter->osdep.flash_bus_space_handle = rman_get_bushandle(adapter->flash); } + /* + ** In the new SPT device flash is not a + ** separate BAR, rather it is also in BAR0, + ** so use the same tag and an offset handle for the + ** FLASH read/write macros in the shared code. + */ + else if (hw->mac.type == e1000_pch_spt) { + adapter->osdep.flash_bus_space_tag = + adapter->osdep.mem_bus_space_tag; + adapter->osdep.flash_bus_space_handle = + adapter->osdep.mem_bus_space_handle + + E1000_FLASH_BASE_ADDR; + } /* Do Shared Code initialization */ - if (e1000_setup_init_funcs(hw, TRUE)) { - device_printf(dev, "Setup of Shared code failed\n"); + error = e1000_setup_init_funcs(hw, TRUE); + if (error) { + device_printf(dev, "Setup of Shared code failed, error %d\n", + error); error = ENXIO; goto err_pci; } + /* + * Setup MSI/X or MSI if PCI Express + */ + adapter->msix = em_setup_msix(adapter); + e1000_get_bus_info(hw); /* Set up some sysctls for the tunable interrupt delays */ @@ -604,7 +659,7 @@ em_attach(device_t dev) } else adapter->num_tx_desc = em_txd; - if (((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 || + if (((em_rxd * sizeof(union e1000_rx_desc_extended)) % EM_DBA_ALIGN) != 0 || (em_rxd > EM_MAX_RXD) || (em_rxd < EM_MIN_RXD)) { device_printf(dev, "Using %d RX descriptors instead of %d!\n", EM_DEFAULT_RXD, em_rxd); @@ -746,8 +801,7 @@ em_attach(device_t dev) em_get_hw_control(adapter); /* Tell the stack that the interface is not active */ - adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; - adapter->ifp->if_drv_flags |= IFF_DRV_OACTIVE; + if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); adapter->led_dev = led_create(em_led_func, adapter, device_get_nameunit(dev)); @@ -763,7 +817,7 @@ err_late: em_free_transmit_structures(adapter); em_free_receive_structures(adapter); em_release_hw_control(adapter); - if (adapter->ifp != NULL) + if (adapter->ifp != (void *)NULL) if_free(adapter->ifp); err_pci: em_free_pci_resources(adapter); @@ -787,18 +841,18 @@ static int em_detach(device_t dev) { struct adapter *adapter = device_get_softc(dev); - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; INIT_DEBUGOUT("em_detach: begin"); /* Make sure VLANS are not using driver */ - if (adapter->ifp->if_vlantrunk != NULL) { + if (if_vlantrunkinuse(ifp)) { device_printf(dev,"Vlan in use, detach first\n"); return (EBUSY); } #ifdef DEVICE_POLLING - if (ifp->if_capenable & IFCAP_POLLING) + if (if_getcapenable(ifp) & IFCAP_POLLING) ether_poll_deregister(ifp); #endif @@ -878,7 +932,7 @@ em_resume(device_t dev) { struct adapter *adapter = device_get_softc(dev); struct tx_ring *txr = adapter->tx_rings; - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; EM_CORE_LOCK(adapter); if (adapter->hw.mac.type == e1000_pch2lan) @@ -886,15 +940,15 @@ em_resume(device_t dev) em_init_locked(adapter); em_init_manageability(adapter); - if ((ifp->if_flags & IFF_UP) && - (ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) { + if ((if_getflags(ifp) & IFF_UP) && + (if_getdrvflags(ifp) & IFF_DRV_RUNNING) && adapter->link_active) { for (int i = 0; i < adapter->num_queues; i++, txr++) { EM_TX_LOCK(txr); #ifdef EM_MULTIQUEUE if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr, NULL); + em_mq_start_locked(ifp, txr); #else - if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) + if (!if_sendq_empty(ifp)) em_start_locked(ifp, txr); #endif EM_TX_UNLOCK(txr); @@ -906,7 +960,70 @@ em_resume(device_t dev) } -#ifdef EM_MULTIQUEUE +#ifndef EM_MULTIQUEUE +static void +em_start_locked(if_t ifp, struct tx_ring *txr) +{ + struct adapter *adapter = if_getsoftc(ifp); + struct mbuf *m_head; + + EM_TX_LOCK_ASSERT(txr); + + if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != + IFF_DRV_RUNNING) + return; + + if (!adapter->link_active) + return; + + while (!if_sendq_empty(ifp)) { + /* Call cleanup if number of TX descriptors low */ + if (txr->tx_avail <= EM_TX_CLEANUP_THRESHOLD) + em_txeof(txr); + if (txr->tx_avail < EM_MAX_SCATTER) { + if_setdrvflagbits(ifp,IFF_DRV_OACTIVE, 0); + break; + } + m_head = if_dequeue(ifp); + if (m_head == NULL) + break; + /* + * Encapsulation can modify our pointer, and or make it + * NULL on failure. In that event, we can't requeue. + */ + if (em_xmit(txr, &m_head)) { + if (m_head == NULL) + break; + if_sendq_prepend(ifp, m_head); + break; + } + + /* Mark the queue as having work */ + if (txr->busy == EM_TX_IDLE) + txr->busy = EM_TX_BUSY; + + /* Send a copy of the frame to the BPF listener */ + ETHER_BPF_MTAP(ifp, m_head); + + } + + return; +} + +static void +em_start(if_t ifp) +{ + struct adapter *adapter = if_getsoftc(ifp); + struct tx_ring *txr = adapter->tx_rings; + + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { + EM_TX_LOCK(txr); + em_start_locked(ifp, txr); + EM_TX_UNLOCK(txr); + } + return; +} +#else /* EM_MULTIQUEUE */ /********************************************************************* * Multiqueue Transmit routines * @@ -915,85 +1032,95 @@ em_resume(device_t dev) * than do an immediate send. It is this that is an advantage * in this driver, rather than also having multiple tx queues. **********************************************************************/ +/* +** Multiqueue capable stack interface +*/ +static int +em_mq_start(if_t ifp, struct mbuf *m) +{ + struct adapter *adapter = if_getsoftc(ifp); + struct tx_ring *txr = adapter->tx_rings; + unsigned int i, error; + + if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) + i = m->m_pkthdr.flowid % adapter->num_queues; + else + i = curcpu % adapter->num_queues; + + txr = &adapter->tx_rings[i]; + + error = drbr_enqueue(ifp, txr->br, m); + if (error) + return (error); + + if (EM_TX_TRYLOCK(txr)) { + em_mq_start_locked(ifp, txr); + EM_TX_UNLOCK(txr); + } else + taskqueue_enqueue(txr->tq, &txr->tx_task); + + return (0); +} + static int -em_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) +em_mq_start_locked(if_t ifp, struct tx_ring *txr) { struct adapter *adapter = txr->adapter; struct mbuf *next; int err = 0, enq = 0; - if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != - IFF_DRV_RUNNING || adapter->link_active == 0) { - if (m != NULL) - err = drbr_enqueue(ifp, txr->br, m); - return (err); - } + EM_TX_LOCK_ASSERT(txr); - enq = 0; - if (m != NULL) { - err = drbr_enqueue(ifp, txr->br, m); - if (err) - return (err); - } + if (((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) || + adapter->link_active == 0) { + return (ENETDOWN); + } /* Process the queue */ while ((next = drbr_peek(ifp, txr->br)) != NULL) { if ((err = em_xmit(txr, &next)) != 0) { - if (next == NULL) + if (next == NULL) { + /* It was freed, move forward */ drbr_advance(ifp, txr->br); - else + } else { + /* + * Still have one left, it may not be + * the same since the transmit function + * may have changed it. + */ drbr_putback(ifp, txr->br, next); + } break; } drbr_advance(ifp, txr->br); enq++; - ifp->if_obytes += next->m_pkthdr.len; + if_inc_counter(ifp, IFCOUNTER_OBYTES, next->m_pkthdr.len); if (next->m_flags & M_MCAST) - ifp->if_omcasts++; + if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); ETHER_BPF_MTAP(ifp, next); - if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) + if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) break; } - if (enq > 0) { - /* Set the watchdog */ - txr->queue_status = EM_QUEUE_WORKING; - txr->watchdog_time = ticks; - } + /* Mark the queue as having work */ + if ((enq > 0) && (txr->busy == EM_TX_IDLE)) + txr->busy = EM_TX_BUSY; if (txr->tx_avail < EM_MAX_SCATTER) em_txeof(txr); - if (txr->tx_avail < EM_MAX_SCATTER) - ifp->if_drv_flags |= IFF_DRV_OACTIVE; + if (txr->tx_avail < EM_MAX_SCATTER) { + if_setdrvflagbits(ifp, IFF_DRV_OACTIVE,0); + } return (err); } /* -** Multiqueue capable stack interface -*/ -static int -em_mq_start(struct ifnet *ifp, struct mbuf *m) -{ - struct adapter *adapter = ifp->if_softc; - struct tx_ring *txr = adapter->tx_rings; - int error; - - if (EM_TX_TRYLOCK(txr)) { - error = em_mq_start_locked(ifp, txr, m); - EM_TX_UNLOCK(txr); - } else - error = drbr_enqueue(ifp, txr->br, m); - - return (error); -} - -/* ** Flush all ring buffers */ static void -em_qflush(struct ifnet *ifp) +em_qflush(if_t ifp) { - struct adapter *adapter = ifp->if_softc; + struct adapter *adapter = if_getsoftc(ifp); struct tx_ring *txr = adapter->tx_rings; struct mbuf *m; @@ -1005,69 +1132,6 @@ em_qflush(struct ifnet *ifp) } if_qflush(ifp); } -#else /* !EM_MULTIQUEUE */ - -static void -em_start_locked(struct ifnet *ifp, struct tx_ring *txr) -{ - struct adapter *adapter = ifp->if_softc; - struct mbuf *m_head; - - EM_TX_LOCK_ASSERT(txr); - - if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != - IFF_DRV_RUNNING) - return; - - if (!adapter->link_active) - return; - - while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { - /* Call cleanup if number of TX descriptors low */ - if (txr->tx_avail <= EM_TX_CLEANUP_THRESHOLD) - em_txeof(txr); - if (txr->tx_avail < EM_MAX_SCATTER) { - ifp->if_drv_flags |= IFF_DRV_OACTIVE; - break; - } - IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); - if (m_head == NULL) - break; - /* - * Encapsulation can modify our pointer, and or make it - * NULL on failure. In that event, we can't requeue. - */ - if (em_xmit(txr, &m_head)) { - if (m_head == NULL) - break; - IFQ_DRV_PREPEND(&ifp->if_snd, m_head); - break; - } - - /* Send a copy of the frame to the BPF listener */ - ETHER_BPF_MTAP(ifp, m_head); - - /* Set timeout in case hardware has problems transmitting. */ - txr->watchdog_time = ticks; - txr->queue_status = EM_QUEUE_WORKING; - } - - return; -} - -static void -em_start(struct ifnet *ifp) -{ - struct adapter *adapter = ifp->if_softc; - struct tx_ring *txr = adapter->tx_rings; - - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - EM_TX_LOCK(txr); - em_start_locked(ifp, txr); - EM_TX_UNLOCK(txr); - } - return; -} #endif /* EM_MULTIQUEUE */ /********************************************************************* @@ -1080,9 +1144,9 @@ em_start(struct ifnet *ifp) **********************************************************************/ static int -em_ioctl(struct ifnet *ifp, u_long command, caddr_t data) +em_ioctl(if_t ifp, u_long command, caddr_t data) { - struct adapter *adapter = ifp->if_softc; + struct adapter *adapter = if_getsoftc(ifp); struct ifreq *ifr = (struct ifreq *)data; #if defined(INET) || defined(INET6) struct ifaddr *ifa = (struct ifaddr *)data; @@ -1108,11 +1172,11 @@ em_ioctl(struct ifnet *ifp, u_long command, caddr_t data) ** so we avoid doing it when possible. */ if (avoid_reset) { - ifp->if_flags |= IFF_UP; - if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) + if_setflagbits(ifp,IFF_UP,0); + if (!(if_getdrvflags(ifp)& IFF_DRV_RUNNING)) em_init(adapter); #ifdef INET - if (!(ifp->if_flags & IFF_NOARP)) + if (!(if_getflags(ifp) & IFF_NOARP)) arp_ifinit(ifp, ifa); #endif } else @@ -1132,6 +1196,7 @@ em_ioctl(struct ifnet *ifp, u_long command, caddr_t data) case e1000_ich10lan: case e1000_pch2lan: case e1000_pch_lpt: + case e1000_pch_spt: case e1000_82574: case e1000_82583: case e1000_80003es2lan: /* 9K Jumbo Frame size */ @@ -1154,10 +1219,11 @@ em_ioctl(struct ifnet *ifp, u_long command, caddr_t data) break; } - ifp->if_mtu = ifr->ifr_mtu; + if_setmtu(ifp, ifr->ifr_mtu); adapter->hw.mac.max_frame_size = - ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; - em_init_locked(adapter); + if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN; + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) + em_init_locked(adapter); EM_CORE_UNLOCK(adapter); break; } @@ -1165,9 +1231,9 @@ em_ioctl(struct ifnet *ifp, u_long command, caddr_t data) IOCTL_DEBUGOUT("ioctl rcv'd:\ SIOCSIFFLAGS (Set Interface Flags)"); EM_CORE_LOCK(adapter); - if (ifp->if_flags & IFF_UP) { - if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { - if ((ifp->if_flags ^ adapter->if_flags) & + if (if_getflags(ifp) & IFF_UP) { + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { + if ((if_getflags(ifp) ^ adapter->if_flags) & (IFF_PROMISC | IFF_ALLMULTI)) { em_disable_promisc(adapter); em_set_promisc(adapter); @@ -1175,20 +1241,20 @@ em_ioctl(struct ifnet *ifp, u_long command, caddr_t data) } else em_init_locked(adapter); } else - if (ifp->if_drv_flags & IFF_DRV_RUNNING) + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) em_stop(adapter); - adapter->if_flags = ifp->if_flags; + adapter->if_flags = if_getflags(ifp); EM_CORE_UNLOCK(adapter); break; case SIOCADDMULTI: case SIOCDELMULTI: IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { EM_CORE_LOCK(adapter); em_disable_intr(adapter); em_set_multi(adapter); #ifdef DEVICE_POLLING - if (!(ifp->if_capenable & IFCAP_POLLING)) + if (!(if_getcapenable(ifp) & IFCAP_POLLING)) #endif em_enable_intr(adapter); EM_CORE_UNLOCK(adapter); @@ -1216,7 +1282,7 @@ em_ioctl(struct ifnet *ifp, u_long command, caddr_t data) IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); reinit = 0; - mask = ifr->ifr_reqcap ^ ifp->if_capenable; + mask = ifr->ifr_reqcap ^ if_getcapenable(ifp); #ifdef DEVICE_POLLING if (mask & IFCAP_POLLING) { if (ifr->ifr_reqcap & IFCAP_POLLING) { @@ -1225,48 +1291,48 @@ em_ioctl(struct ifnet *ifp, u_long command, caddr_t data) return (error); EM_CORE_LOCK(adapter); em_disable_intr(adapter); - ifp->if_capenable |= IFCAP_POLLING; + if_setcapenablebit(ifp, IFCAP_POLLING, 0); EM_CORE_UNLOCK(adapter); } else { error = ether_poll_deregister(ifp); /* Enable interrupt even in error case */ EM_CORE_LOCK(adapter); em_enable_intr(adapter); - ifp->if_capenable &= ~IFCAP_POLLING; + if_setcapenablebit(ifp, 0, IFCAP_POLLING); EM_CORE_UNLOCK(adapter); } } #endif if (mask & IFCAP_HWCSUM) { - ifp->if_capenable ^= IFCAP_HWCSUM; + if_togglecapenable(ifp,IFCAP_HWCSUM); reinit = 1; } if (mask & IFCAP_TSO4) { - ifp->if_capenable ^= IFCAP_TSO4; + if_togglecapenable(ifp,IFCAP_TSO4); reinit = 1; } if (mask & IFCAP_VLAN_HWTAGGING) { - ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; + if_togglecapenable(ifp,IFCAP_VLAN_HWTAGGING); reinit = 1; } if (mask & IFCAP_VLAN_HWFILTER) { - ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; + if_togglecapenable(ifp, IFCAP_VLAN_HWFILTER); reinit = 1; } if (mask & IFCAP_VLAN_HWTSO) { - ifp->if_capenable ^= IFCAP_VLAN_HWTSO; + if_togglecapenable(ifp, IFCAP_VLAN_HWTSO); reinit = 1; } if ((mask & IFCAP_WOL) && - (ifp->if_capabilities & IFCAP_WOL) != 0) { + (if_getcapabilities(ifp) & IFCAP_WOL) != 0) { if (mask & IFCAP_WOL_MCAST) - ifp->if_capenable ^= IFCAP_WOL_MCAST; + if_togglecapenable(ifp, IFCAP_WOL_MCAST); if (mask & IFCAP_WOL_MAGIC) - ifp->if_capenable ^= IFCAP_WOL_MAGIC; + if_togglecapenable(ifp, IFCAP_WOL_MAGIC); } - if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) + if (reinit && (if_getdrvflags(ifp) & IFF_DRV_RUNNING)) em_init(adapter); - VLAN_CAPABILITIES(ifp); + if_vlancap(ifp); break; } @@ -1293,7 +1359,7 @@ em_ioctl(struct ifnet *ifp, u_long command, caddr_t data) static void em_init_locked(struct adapter *adapter) { - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; device_t dev = adapter->dev; INIT_DEBUGOUT("em_init: begin"); @@ -1304,7 +1370,7 @@ em_init_locked(struct adapter *adapter) callout_stop(&adapter->timer); /* Get the latest mac address, User can use a LAA */ - bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr, + bcopy(if_getlladdr(adapter->ifp), adapter->hw.mac.addr, ETHER_ADDR_LEN); /* Put the address into the Receive Address Array */ @@ -1330,11 +1396,18 @@ em_init_locked(struct adapter *adapter) E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); /* Set hardware offload abilities */ - ifp->if_hwassist = 0; - if (ifp->if_capenable & IFCAP_TXCSUM) - ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); - if (ifp->if_capenable & IFCAP_TSO4) - ifp->if_hwassist |= CSUM_TSO; + if_clearhwassist(ifp); + if (if_getcapenable(ifp) & IFCAP_TXCSUM) + if_sethwassistbits(ifp, CSUM_TCP | CSUM_UDP, 0); + /* + ** There have proven to be problems with TSO when not + ** at full gigabit speed, so disable the assist automatically + ** when at lower speeds. -jfv + */ + if (if_getcapenable(ifp) & IFCAP_TSO4) { + if (adapter->link_speed == SPEED_1000) + if_sethwassistbits(ifp, CSUM_TSO, 0); + } /* Configure for OS presence */ em_init_manageability(adapter); @@ -1366,8 +1439,8 @@ em_init_locked(struct adapter *adapter) em_initialize_receive_unit(adapter); /* Use real VLAN Filter support? */ - if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) { - if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) + if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) { + if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) /* Use real VLAN Filter support */ em_setup_vlan_hw_support(adapter); else { @@ -1382,8 +1455,7 @@ em_init_locked(struct adapter *adapter) em_set_promisc(adapter); /* Set the interface as ACTIVE */ - ifp->if_drv_flags |= IFF_DRV_RUNNING; - ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; + if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); callout_reset(&adapter->timer, hz, em_local_timer, adapter); e1000_clear_hw_cntrs_base_generic(&adapter->hw); @@ -1403,7 +1475,7 @@ em_init_locked(struct adapter *adapter) * Only enable interrupts if we are not polling, make sure * they are off otherwise. */ - if (ifp->if_capenable & IFCAP_POLLING) + if (if_getcapenable(ifp) & IFCAP_POLLING) em_disable_intr(adapter); else #endif /* DEVICE_POLLING */ @@ -1432,16 +1504,16 @@ em_init(void *arg) * *********************************************************************/ static int -em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) +em_poll(if_t ifp, enum poll_cmd cmd, int count) { - struct adapter *adapter = ifp->if_softc; + struct adapter *adapter = if_getsoftc(ifp); struct tx_ring *txr = adapter->tx_rings; struct rx_ring *rxr = adapter->rx_rings; u32 reg_icr; int rx_done; EM_CORE_LOCK(adapter); - if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { + if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) { EM_CORE_UNLOCK(adapter); return (0); } @@ -1464,9 +1536,9 @@ em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) em_txeof(txr); #ifdef EM_MULTIQUEUE if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr, NULL); + em_mq_start_locked(ifp, txr); #else - if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) + if (!if_sendq_empty(ifp)) em_start_locked(ifp, txr); #endif EM_TX_UNLOCK(txr); @@ -1485,7 +1557,7 @@ static int em_irq_fast(void *arg) { struct adapter *adapter = arg; - struct ifnet *ifp; + if_t ifp; u32 reg_icr; ifp = adapter->ifp; @@ -1527,20 +1599,20 @@ static void em_handle_que(void *context, int pending) { struct adapter *adapter = context; - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; struct tx_ring *txr = adapter->tx_rings; struct rx_ring *rxr = adapter->rx_rings; - - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { bool more = em_rxeof(rxr, adapter->rx_process_limit, NULL); + EM_TX_LOCK(txr); em_txeof(txr); #ifdef EM_MULTIQUEUE if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr, NULL); + em_mq_start_locked(ifp, txr); #else - if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) + if (!if_sendq_empty(ifp)) em_start_locked(ifp, txr); #endif EM_TX_UNLOCK(txr); @@ -1565,18 +1637,19 @@ em_msix_tx(void *arg) { struct tx_ring *txr = arg; struct adapter *adapter = txr->adapter; - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; ++txr->tx_irq; EM_TX_LOCK(txr); em_txeof(txr); #ifdef EM_MULTIQUEUE if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr, NULL); + em_mq_start_locked(ifp, txr); #else - if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) + if (!if_sendq_empty(ifp)) em_start_locked(ifp, txr); #endif + /* Reenable this interrupt */ E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims); EM_TX_UNLOCK(txr); @@ -1597,14 +1670,15 @@ em_msix_rx(void *arg) bool more; ++rxr->rx_irq; - if (!(adapter->ifp->if_drv_flags & IFF_DRV_RUNNING)) + if (!(if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING)) return; more = em_rxeof(rxr, adapter->rx_process_limit, NULL); if (more) taskqueue_enqueue(rxr->tq, &rxr->rx_task); - else + else { /* Reenable this interrupt */ E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims); + } return; } @@ -1622,12 +1696,25 @@ em_msix_link(void *arg) ++adapter->link_irq; reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); + if (reg_icr & E1000_ICR_RXO) + adapter->rx_overruns++; + if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { adapter->hw.mac.get_link_status = 1; em_handle_link(adapter, 0); } else E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_LINK | E1000_IMS_LSC); + /* + ** Because we must read the ICR for this interrupt + ** it may clear other causes using autoclear, for + ** this reason we simply create a soft interrupt + ** for all these vectors. + */ + if (reg_icr) { + E1000_WRITE_REG(&adapter->hw, + E1000_ICS, adapter->ims); + } return; } @@ -1641,9 +1728,10 @@ em_handle_rx(void *context, int pending) more = em_rxeof(rxr, adapter->rx_process_limit, NULL); if (more) taskqueue_enqueue(rxr->tq, &rxr->rx_task); - else + else { /* Reenable this interrupt */ E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims); + } } static void @@ -1651,15 +1739,15 @@ em_handle_tx(void *context, int pending) { struct tx_ring *txr = context; struct adapter *adapter = txr->adapter; - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; EM_TX_LOCK(txr); em_txeof(txr); #ifdef EM_MULTIQUEUE if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr, NULL); + em_mq_start_locked(ifp, txr); #else - if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) + if (!if_sendq_empty(ifp)) em_start_locked(ifp, txr); #endif E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims); @@ -1671,9 +1759,9 @@ em_handle_link(void *context, int pending) { struct adapter *adapter = context; struct tx_ring *txr = adapter->tx_rings; - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; - if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) + if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) return; EM_CORE_LOCK(adapter); @@ -1687,9 +1775,9 @@ em_handle_link(void *context, int pending) EM_TX_LOCK(txr); #ifdef EM_MULTIQUEUE if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr, NULL); + em_mq_start_locked(ifp, txr); #else - if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) + if (if_sendq_empty(ifp)) em_start_locked(ifp, txr); #endif EM_TX_UNLOCK(txr); @@ -1708,9 +1796,9 @@ em_handle_link(void *context, int pending) * **********************************************************************/ static void -em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) +em_media_status(if_t ifp, struct ifmediareq *ifmr) { - struct adapter *adapter = ifp->if_softc; + struct adapter *adapter = if_getsoftc(ifp); u_char fiber_type = IFM_1000_SX; INIT_DEBUGOUT("em_media_status: begin"); @@ -1760,9 +1848,9 @@ em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) * **********************************************************************/ static int -em_media_change(struct ifnet *ifp) +em_media_change(if_t ifp) { - struct adapter *adapter = ifp->if_softc; + struct adapter *adapter = if_getsoftc(ifp); struct ifmedia *ifm = &adapter->media; INIT_DEBUGOUT("em_media_change: begin"); @@ -1821,20 +1909,21 @@ em_xmit(struct tx_ring *txr, struct mbuf **m_headp) struct adapter *adapter = txr->adapter; bus_dma_segment_t segs[EM_MAX_SCATTER]; bus_dmamap_t map; - struct em_buffer *tx_buffer, *tx_buffer_mapped; + struct em_txbuffer *tx_buffer, *tx_buffer_mapped; struct e1000_tx_desc *ctxd = NULL; struct mbuf *m_head; struct ether_header *eh; struct ip *ip = NULL; struct tcphdr *tp = NULL; - u32 txd_upper, txd_lower, txd_used, txd_saved; + u32 txd_upper = 0, txd_lower = 0; int ip_off, poff; int nsegs, i, j, first, last = 0; - int error, do_tso, tso_desc = 0, remap = 1; + int error; + bool do_tso, tso_desc, remap = TRUE; m_head = *m_headp; - txd_upper = txd_lower = txd_used = txd_saved = 0; - do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0); + do_tso = (m_head->m_pkthdr.csum_flags & CSUM_TSO); + tso_desc = FALSE; ip_off = poff = 0; /* @@ -1849,7 +1938,7 @@ em_xmit(struct tx_ring *txr, struct mbuf **m_headp) * so we firstly get a writable mbuf chain then coalesce ethernet/ * IP/TCP header into a single buffer to meet the requirement of * controller. This also simplifies IP/TCP/UDP checksum offloading - * which also has similiar restrictions. + * which also has similar restrictions. */ if (do_tso || m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) { if (do_tso || (m_head->m_next != NULL && @@ -1870,74 +1959,82 @@ em_xmit(struct tx_ring *txr, struct mbuf **m_headp) * for IPv6 yet. */ ip_off = sizeof(struct ether_header); - m_head = m_pullup(m_head, ip_off); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); + if (m_head->m_len < ip_off) { + m_head = m_pullup(m_head, ip_off); + if (m_head == NULL) { + *m_headp = NULL; + return (ENOBUFS); + } } eh = mtod(m_head, struct ether_header *); if (eh->ether_type == htons(ETHERTYPE_VLAN)) { ip_off = sizeof(struct ether_vlan_header); - m_head = m_pullup(m_head, ip_off); + if (m_head->m_len < ip_off) { + m_head = m_pullup(m_head, ip_off); + if (m_head == NULL) { + *m_headp = NULL; + return (ENOBUFS); + } + } + } + if (m_head->m_len < ip_off + sizeof(struct ip)) { + m_head = m_pullup(m_head, ip_off + sizeof(struct ip)); if (m_head == NULL) { *m_headp = NULL; return (ENOBUFS); } } - m_head = m_pullup(m_head, ip_off + sizeof(struct ip)); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); - } ip = (struct ip *)(mtod(m_head, char *) + ip_off); poff = ip_off + (ip->ip_hl << 2); - if (do_tso) { - m_head = m_pullup(m_head, poff + sizeof(struct tcphdr)); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); + + if (do_tso || (m_head->m_pkthdr.csum_flags & CSUM_TCP)) { + if (m_head->m_len < poff + sizeof(struct tcphdr)) { + m_head = m_pullup(m_head, poff + + sizeof(struct tcphdr)); + if (m_head == NULL) { + *m_headp = NULL; + return (ENOBUFS); + } } tp = (struct tcphdr *)(mtod(m_head, char *) + poff); /* * TSO workaround: * pull 4 more bytes of data into it. */ - m_head = m_pullup(m_head, poff + (tp->th_off << 2) + 4); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); + if (m_head->m_len < poff + (tp->th_off << 2)) { + m_head = m_pullup(m_head, poff + + (tp->th_off << 2) + + TSO_WORKAROUND); + if (m_head == NULL) { + *m_headp = NULL; + return (ENOBUFS); + } } ip = (struct ip *)(mtod(m_head, char *) + ip_off); - ip->ip_len = 0; - ip->ip_sum = 0; - /* - * The pseudo TCP checksum does not include TCP payload - * length so driver should recompute the checksum here - * what hardware expect to see. This is adherence of - * Microsoft's Large Send specification. - */ tp = (struct tcphdr *)(mtod(m_head, char *) + poff); - tp->th_sum = in_pseudo(ip->ip_src.s_addr, - ip->ip_dst.s_addr, htons(IPPROTO_TCP)); - } else if (m_head->m_pkthdr.csum_flags & CSUM_TCP) { - m_head = m_pullup(m_head, poff + sizeof(struct tcphdr)); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); - } - tp = (struct tcphdr *)(mtod(m_head, char *) + poff); - m_head = m_pullup(m_head, poff + (tp->th_off << 2)); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); + if (do_tso) { + ip->ip_len = htons(m_head->m_pkthdr.tso_segsz + + (ip->ip_hl << 2) + + (tp->th_off << 2)); + ip->ip_sum = 0; + /* + * The pseudo TCP checksum does not include TCP + * payload length so driver should recompute + * the checksum here what hardware expect to + * see. This is adherence of Microsoft's Large + * Send specification. + */ + tp->th_sum = in_pseudo(ip->ip_src.s_addr, + ip->ip_dst.s_addr, htons(IPPROTO_TCP)); } - ip = (struct ip *)(mtod(m_head, char *) + ip_off); - tp = (struct tcphdr *)(mtod(m_head, char *) + poff); } else if (m_head->m_pkthdr.csum_flags & CSUM_UDP) { - m_head = m_pullup(m_head, poff + sizeof(struct udphdr)); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); + if (m_head->m_len < poff + sizeof(struct udphdr)) { + m_head = m_pullup(m_head, poff + + sizeof(struct udphdr)); + if (m_head == NULL) { + *m_headp = NULL; + return (ENOBUFS); + } } ip = (struct ip *)(mtod(m_head, char *) + ip_off); } @@ -1973,9 +2070,9 @@ retry: if (error == EFBIG && remap) { struct mbuf *m; - m = m_defrag(*m_headp, M_NOWAIT); + m = m_collapse(*m_headp, M_NOWAIT, EM_MAX_SCATTER); if (m == NULL) { - adapter->mbuf_alloc_failed++; + adapter->mbuf_defrag_failed++; m_freem(*m_headp); *m_headp = NULL; return (ENOBUFS); @@ -1983,11 +2080,8 @@ retry: *m_headp = m; /* Try it again, but only once */ - remap = 0; + remap = FALSE; goto retry; - } else if (error == ENOMEM) { - adapter->no_tx_dma_setup++; - return (error); } else if (error != 0) { adapter->no_tx_dma_setup++; m_freem(*m_headp); @@ -2001,13 +2095,13 @@ retry: * it follows a TSO burst, then we need to add a * sentinel descriptor to prevent premature writeback. */ - if ((do_tso == 0) && (txr->tx_tso == TRUE)) { + if ((!do_tso) && (txr->tx_tso == TRUE)) { if (nsegs == 1) tso_desc = TRUE; txr->tx_tso = FALSE; } - if (nsegs > (txr->tx_avail - 2)) { + if (txr->tx_avail < (nsegs + EM_MAX_SCATTER)) { txr->no_desc_avail++; bus_dmamap_unload(txr->txtag, map); return (ENOBUFS); @@ -2026,8 +2120,7 @@ retry: if (m_head->m_flags & M_VLANTAG) { /* Set the vlan id. */ - txd_upper |= - (htole16(m_head->m_pkthdr.ether_vtag) << 16); + txd_upper |= htole16(if_getvtag(m_head)) << 16; /* Tell hardware to add tag */ txd_lower |= htole32(E1000_TXD_CMD_VLE); } @@ -2048,23 +2141,23 @@ retry: ** If this is the last descriptor, we want to ** split it so we have a small final sentinel */ - if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) { - seg_len -= 4; + if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) { + seg_len -= TSO_WORKAROUND; ctxd->buffer_addr = htole64(seg_addr); ctxd->lower.data = htole32( - adapter->txd_cmd | txd_lower | seg_len); - ctxd->upper.data = - htole32(txd_upper); + adapter->txd_cmd | txd_lower | seg_len); + ctxd->upper.data = htole32(txd_upper); if (++i == adapter->num_tx_desc) i = 0; + /* Now make the sentinel */ - ++txd_used; /* using an extra txd */ + txr->tx_avail--; ctxd = &txr->tx_base[i]; tx_buffer = &txr->tx_buffers[i]; ctxd->buffer_addr = htole64(seg_addr + seg_len); ctxd->lower.data = htole32( - adapter->txd_cmd | txd_lower | 4); + adapter->txd_cmd | txd_lower | TSO_WORKAROUND); ctxd->upper.data = htole32(txd_upper); last = i; @@ -2074,8 +2167,7 @@ retry: ctxd->buffer_addr = htole64(seg_addr); ctxd->lower.data = htole32( adapter->txd_cmd | txd_lower | seg_len); - ctxd->upper.data = - htole32(txd_upper); + ctxd->upper.data = htole32(txd_upper); last = i; if (++i == adapter->num_tx_desc) i = 0; @@ -2086,8 +2178,6 @@ retry: txr->next_avail_desc = i; txr->tx_avail -= nsegs; - if (tso_desc) /* TSO used an extra for sentinel */ - txr->tx_avail -= txd_used; tx_buffer->m_head = m_head; /* @@ -2113,8 +2203,6 @@ retry: */ tx_buffer = &txr->tx_buffers[first]; tx_buffer->next_eop = last; - /* Update the watchdog time early and often */ - txr->watchdog_time = ticks; /* * Advance the Transmit Descriptor Tail (TDT), this tells the E1000 @@ -2130,18 +2218,18 @@ retry: static void em_set_promisc(struct adapter *adapter) { - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; u32 reg_rctl; reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); - if (ifp->if_flags & IFF_PROMISC) { + if (if_getflags(ifp) & IFF_PROMISC) { reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); /* Turn this on if you want to see bad packets */ if (em_debug_sbp) reg_rctl |= E1000_RCTL_SBP; E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); - } else if (ifp->if_flags & IFF_ALLMULTI) { + } else if (if_getflags(ifp) & IFF_ALLMULTI) { reg_rctl |= E1000_RCTL_MPE; reg_rctl &= ~E1000_RCTL_UPE; E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); @@ -2151,34 +2239,16 @@ em_set_promisc(struct adapter *adapter) static void em_disable_promisc(struct adapter *adapter) { - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; u32 reg_rctl; int mcnt = 0; reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); reg_rctl &= (~E1000_RCTL_UPE); - if (ifp->if_flags & IFF_ALLMULTI) + if (if_getflags(ifp) & IFF_ALLMULTI) mcnt = MAX_NUM_MULTICAST_ADDRESSES; - else { - struct ifmultiaddr *ifma; -#if __FreeBSD_version < 800000 - IF_ADDR_LOCK(ifp); -#else - if_maddr_rlock(ifp); -#endif - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { - if (ifma->ifma_addr->sa_family != AF_LINK) - continue; - if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) - break; - mcnt++; - } -#if __FreeBSD_version < 800000 - IF_ADDR_UNLOCK(ifp); -#else - if_maddr_runlock(ifp); -#endif - } + else + mcnt = if_multiaddr_count(ifp, MAX_NUM_MULTICAST_ADDRESSES); /* Don't disable if in MAX groups */ if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) reg_rctl &= (~E1000_RCTL_MPE); @@ -2197,8 +2267,7 @@ em_disable_promisc(struct adapter *adapter) static void em_set_multi(struct adapter *adapter) { - struct ifnet *ifp = adapter->ifp; - struct ifmultiaddr *ifma; + if_t ifp = adapter->ifp; u32 reg_rctl = 0; u8 *mta; /* Multicast array memory */ int mcnt = 0; @@ -2218,27 +2287,8 @@ em_set_multi(struct adapter *adapter) msec_delay(5); } -#if __FreeBSD_version < 800000 - IF_ADDR_LOCK(ifp); -#else - if_maddr_rlock(ifp); -#endif - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { - if (ifma->ifma_addr->sa_family != AF_LINK) - continue; - - if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) - break; + if_multiaddr_array(ifp, mta, &mcnt, MAX_NUM_MULTICAST_ADDRESSES); - bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), - &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN); - mcnt++; - } -#if __FreeBSD_version < 800000 - IF_ADDR_UNLOCK(ifp); -#else - if_maddr_runlock(ifp); -#endif if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) { reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); reg_rctl |= E1000_RCTL_MPE; @@ -2269,10 +2319,10 @@ static void em_local_timer(void *arg) { struct adapter *adapter = arg; - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; struct tx_ring *txr = adapter->tx_rings; struct rx_ring *rxr = adapter->rx_rings; - u32 trigger; + u32 trigger = 0; EM_CORE_LOCK_ASSERT(adapter); @@ -2285,9 +2335,11 @@ em_local_timer(void *arg) e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); /* Mask to use in the irq trigger */ - if (adapter->msix_mem) - trigger = rxr->ims; - else + if (adapter->msix_mem) { + for (int i = 0; i < adapter->num_queues; i++, rxr++) + trigger |= rxr->ims; + rxr = adapter->rx_rings; + } else trigger = E1000_ICS_RXDMT0; /* @@ -2296,15 +2348,15 @@ em_local_timer(void *arg) ** and the HUNG state will be static if set. */ for (int i = 0; i < adapter->num_queues; i++, txr++) { - if ((txr->queue_status == EM_QUEUE_HUNG) && - (adapter->pause_frames == 0)) + if (txr->busy == EM_TX_HUNG) goto hung; + if (txr->busy >= EM_TX_MAXTRIES) + txr->busy = EM_TX_HUNG; /* Schedule a TX tasklet if needed */ if (txr->tx_avail <= EM_MAX_SCATTER) taskqueue_enqueue(txr->tq, &txr->tx_task); } - adapter->pause_frames = 0; callout_reset(&adapter->timer, hz, em_local_timer, adapter); #ifndef DEVICE_POLLING /* Trigger an RX interrupt to guarantee mbuf refresh */ @@ -2313,17 +2365,11 @@ em_local_timer(void *arg) return; hung: /* Looks like we're hung */ - device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); - device_printf(adapter->dev, - "Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, - E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)), - E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me))); - device_printf(adapter->dev,"TX(%d) desc avail = %d," - "Next TX to Clean = %d\n", - txr->me, txr->tx_avail, txr->next_to_clean); - ifp->if_drv_flags &= ~IFF_DRV_RUNNING; + device_printf(adapter->dev, "Watchdog timeout Queue[%d]-- resetting\n", + txr->me); + em_print_debug_info(adapter); + if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); adapter->watchdog_events++; - adapter->pause_frames = 0; em_init_locked(adapter); } @@ -2332,7 +2378,7 @@ static void em_update_link_status(struct adapter *adapter) { struct e1000_hw *hw = &adapter->hw; - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; device_t dev = adapter->dev; struct tx_ring *txr = adapter->tx_rings; u32 link_check = 0; @@ -2341,6 +2387,8 @@ em_update_link_status(struct adapter *adapter) switch (hw->phy.media_type) { case e1000_media_type_copper: if (hw->mac.get_link_status) { + if (hw->mac.type == e1000_pch_spt) + msec_delay(50); /* Do the work to read phy */ e1000_check_for_link(hw); link_check = !hw->mac.get_link_status; @@ -2373,7 +2421,7 @@ em_update_link_status(struct adapter *adapter) (hw->mac.type == e1000_82572))) { int tarc0; tarc0 = E1000_READ_REG(hw, E1000_TARC(0)); - tarc0 &= ~SPEED_MODE_BIT; + tarc0 &= ~TARC_SPEED_MODE_BIT; E1000_WRITE_REG(hw, E1000_TARC(0), tarc0); } if (bootverbose) @@ -2383,17 +2431,18 @@ em_update_link_status(struct adapter *adapter) "Full Duplex" : "Half Duplex")); adapter->link_active = 1; adapter->smartspeed = 0; - ifp->if_baudrate = adapter->link_speed * 1000000; + if_setbaudrate(ifp, adapter->link_speed * 1000000); if_link_state_change(ifp, LINK_STATE_UP); } else if (!link_check && (adapter->link_active == 1)) { - ifp->if_baudrate = adapter->link_speed = 0; + if_setbaudrate(ifp, 0); + adapter->link_speed = 0; adapter->link_duplex = 0; if (bootverbose) device_printf(dev, "Link is Down\n"); adapter->link_active = 0; - /* Link down, disable watchdog */ + /* Link down, disable hang detection */ for (int i = 0; i < adapter->num_queues; i++, txr++) - txr->queue_status = EM_QUEUE_IDLE; + txr->busy = EM_TX_IDLE; if_link_state_change(ifp, LINK_STATE_DOWN); } } @@ -2411,7 +2460,7 @@ static void em_stop(void *arg) { struct adapter *adapter = arg; - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; struct tx_ring *txr = adapter->tx_rings; EM_CORE_LOCK_ASSERT(adapter); @@ -2422,16 +2471,19 @@ em_stop(void *arg) callout_stop(&adapter->timer); /* Tell the stack that the interface is no longer active */ - ifp->if_drv_flags &= ~IFF_DRV_RUNNING; - ifp->if_drv_flags |= IFF_DRV_OACTIVE; + if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); - /* Unarm watchdog timer. */ + /* Disarm Hang Detection. */ for (int i = 0; i < adapter->num_queues; i++, txr++) { EM_TX_LOCK(txr); - txr->queue_status = EM_QUEUE_IDLE; + txr->busy = EM_TX_IDLE; EM_TX_UNLOCK(txr); } + /* I219 needs some special flushing to avoid hangs */ + if (adapter->hw.mac.type == e1000_pch_spt) + em_flush_desc_rings(adapter); + e1000_reset_hw(&adapter->hw); E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); @@ -2489,14 +2541,6 @@ em_allocate_pci_resources(struct adapter *adapter) rman_get_bushandle(adapter->memory); adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; - /* Default to a single queue */ - adapter->num_queues = 1; - - /* - * Setup MSI/X or MSI if PCI Express - */ - adapter->msix = em_setup_msix(adapter); - adapter->hw.back = &adapter->osdep; return (0); @@ -2560,7 +2604,7 @@ em_allocate_legacy(struct adapter *adapter) * * Setup the MSIX Interrupt handlers * This is not really Multiqueue, rather - * its just seperate interrupt vectors + * its just separate interrupt vectors * for TX, RX, and Link. * **********************************************************************/ @@ -2571,13 +2615,14 @@ em_allocate_msix(struct adapter *adapter) struct tx_ring *txr = adapter->tx_rings; struct rx_ring *rxr = adapter->rx_rings; int error, rid, vector = 0; + int cpu_id = 0; /* Make sure all interrupts are disabled */ E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); /* First set up ring resources */ - for (int i = 0; i < adapter->num_queues; i++, txr++, rxr++) { + for (int i = 0; i < adapter->num_queues; i++, rxr++, vector++) { /* RX ring */ rid = vector + 1; @@ -2597,14 +2642,20 @@ em_allocate_msix(struct adapter *adapter) return (error); } #if __FreeBSD_version >= 800504 - bus_describe_intr(dev, rxr->res, rxr->tag, "rx %d", i); + bus_describe_intr(dev, rxr->res, rxr->tag, "rx%d", i); #endif - rxr->msix = vector++; /* NOTE increment vector for TX */ + rxr->msix = vector; + + if (em_last_bind_cpu < 0) + em_last_bind_cpu = CPU_FIRST(); + cpu_id = em_last_bind_cpu; + bus_bind_intr(dev, rxr->res, cpu_id); + TASK_INIT(&rxr->rx_task, 0, em_handle_rx, rxr); rxr->tq = taskqueue_create_fast("em_rxq", M_NOWAIT, taskqueue_thread_enqueue, &rxr->tq); - taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq", - device_get_nameunit(adapter->dev)); + taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq (cpuid %d)", + device_get_nameunit(adapter->dev), cpu_id); /* ** Set the bit to enable interrupt ** in E1000_IMS -- bits 20 and 21 @@ -2612,8 +2663,13 @@ em_allocate_msix(struct adapter *adapter) ** NOTHING to do with the MSIX vector */ rxr->ims = 1 << (20 + i); + adapter->ims |= rxr->ims; adapter->ivars |= (8 | rxr->msix) << (i * 4); + em_last_bind_cpu = CPU_NEXT(em_last_bind_cpu); + } + + for (int i = 0; i < adapter->num_queues; i++, txr++, vector++) { /* TX ring */ rid = vector + 1; txr->res = bus_alloc_resource_any(dev, @@ -2631,14 +2687,20 @@ em_allocate_msix(struct adapter *adapter) return (error); } #if __FreeBSD_version >= 800504 - bus_describe_intr(dev, txr->res, txr->tag, "tx %d", i); + bus_describe_intr(dev, txr->res, txr->tag, "tx%d", i); #endif - txr->msix = vector++; /* Increment vector for next pass */ + txr->msix = vector; + + if (em_last_bind_cpu < 0) + em_last_bind_cpu = CPU_FIRST(); + cpu_id = em_last_bind_cpu; + bus_bind_intr(dev, txr->res, cpu_id); + TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr); txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT, taskqueue_thread_enqueue, &txr->tq); - taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq", - device_get_nameunit(adapter->dev)); + taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq (cpuid %d)", + device_get_nameunit(adapter->dev), cpu_id); /* ** Set the bit to enable interrupt ** in E1000_IMS -- bits 22 and 23 @@ -2646,13 +2708,16 @@ em_allocate_msix(struct adapter *adapter) ** NOTHING to do with the MSIX vector */ txr->ims = 1 << (22 + i); + adapter->ims |= txr->ims; adapter->ivars |= (8 | txr->msix) << (8 + (i * 4)); + + em_last_bind_cpu = CPU_NEXT(em_last_bind_cpu); } /* Link interrupt */ - ++rid; + rid = vector + 1; adapter->res = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &rid, RF_ACTIVE); + SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); if (!adapter->res) { device_printf(dev,"Unable to allocate " "bus resource: Link interrupt [%d]\n", rid); @@ -2668,7 +2733,7 @@ em_allocate_msix(struct adapter *adapter) return (error); } #if __FreeBSD_version >= 800504 - bus_describe_intr(dev, adapter->res, adapter->tag, "link"); + bus_describe_intr(dev, adapter->res, adapter->tag, "link"); #endif adapter->linkvec = vector; adapter->ivars |= (8 | vector) << 16; @@ -2692,9 +2757,8 @@ em_free_pci_resources(struct adapter *adapter) */ for (int i = 0; i < adapter->num_queues; i++) { txr = &adapter->tx_rings[i]; - rxr = &adapter->rx_rings[i]; /* an early abort? */ - if ((txr == NULL) || (rxr == NULL)) + if (txr == NULL) break; rid = txr->msix +1; if (txr->tag != NULL) { @@ -2704,6 +2768,11 @@ em_free_pci_resources(struct adapter *adapter) if (txr->res != NULL) bus_release_resource(dev, SYS_RES_IRQ, rid, txr->res); + + rxr = &adapter->rx_rings[i]; + /* an early abort? */ + if (rxr == NULL) + break; rid = rxr->msix +1; if (rxr->tag != NULL) { bus_teardown_intr(dev, rxr->res, rxr->tag); @@ -2753,14 +2822,19 @@ em_setup_msix(struct adapter *adapter) device_t dev = adapter->dev; int val; + /* Nearly always going to use one queue */ + adapter->num_queues = 1; + /* - ** Setup MSI/X for Hartwell: tests have shown - ** use of two queues to be unstable, and to - ** provide no great gain anyway, so we simply - ** seperate the interrupts and use a single queue. + ** Try using MSI-X for Hartwell adapters */ if ((adapter->hw.mac.type == e1000_82574) && (em_enable_msix == TRUE)) { +#ifdef EM_MULTIQUEUE + adapter->num_queues = (em_num_queues == 1) ? 1 : 2; + if (adapter->num_queues > 1) + em_enable_vectors_82574(adapter); +#endif /* Map the MSIX BAR */ int rid = PCIR_BAR(EM_MSIX_BAR); adapter->msix_mem = bus_alloc_resource_any(dev, @@ -2772,16 +2846,34 @@ em_setup_msix(struct adapter *adapter) goto msi; } val = pci_msix_count(dev); - /* We only need/want 3 vectors */ - if (val >= 3) - val = 3; - else { - device_printf(adapter->dev, - "MSIX: insufficient vectors, using MSI\n"); - goto msi; + +#ifdef EM_MULTIQUEUE + /* We need 5 vectors in the multiqueue case */ + if (adapter->num_queues > 1 ) { + if (val >= 5) + val = 5; + else { + adapter->num_queues = 1; + device_printf(adapter->dev, + "Insufficient MSIX vectors for >1 queue, " + "using single queue...\n"); + goto msix_one; + } + } else { +msix_one: +#endif + if (val >= 3) + val = 3; + else { + device_printf(adapter->dev, + "Insufficient MSIX vectors, using MSI\n"); + goto msi; + } +#ifdef EM_MULTIQUEUE } +#endif - if ((pci_alloc_msix(dev, &val) == 0) && (val == 3)) { + if ((pci_alloc_msix(dev, &val) == 0)) { device_printf(adapter->dev, "Using MSIX interrupts " "with %d vectors\n", val); @@ -2802,7 +2894,7 @@ msi: } val = 1; if (pci_alloc_msi(dev, &val) == 0) { - device_printf(adapter->dev,"Using an MSI interrupt\n"); + device_printf(adapter->dev, "Using an MSI interrupt\n"); return (val); } /* Should only happen due to manual configuration */ @@ -2811,6 +2903,116 @@ msi: } +/* +** The 3 following flush routines are used as a workaround in the +** I219 client parts and only for them. +** +** em_flush_tx_ring - remove all descriptors from the tx_ring +** +** We want to clear all pending descriptors from the TX ring. +** zeroing happens when the HW reads the regs. We assign the ring itself as +** the data of the next descriptor. We don't care about the data we are about +** to reset the HW. +*/ +static void +em_flush_tx_ring(struct adapter *adapter) +{ + struct e1000_hw *hw = &adapter->hw; + struct tx_ring *txr = adapter->tx_rings; + struct e1000_tx_desc *txd; + u32 tctl, txd_lower = E1000_TXD_CMD_IFCS; + u16 size = 512; + + tctl = E1000_READ_REG(hw, E1000_TCTL); + E1000_WRITE_REG(hw, E1000_TCTL, tctl | E1000_TCTL_EN); + + txd = &txr->tx_base[txr->next_avail_desc++]; + if (txr->next_avail_desc == adapter->num_tx_desc) + txr->next_avail_desc = 0; + + /* Just use the ring as a dummy buffer addr */ + txd->buffer_addr = txr->txdma.dma_paddr; + txd->lower.data = htole32(txd_lower | size); + txd->upper.data = 0; + + /* flush descriptors to memory before notifying the HW */ + wmb(); + + E1000_WRITE_REG(hw, E1000_TDT(0), txr->next_avail_desc); + mb(); + usec_delay(250); +} + +/* +** em_flush_rx_ring - remove all descriptors from the rx_ring +** +** Mark all descriptors in the RX ring as consumed and disable the rx ring +*/ +static void +em_flush_rx_ring(struct adapter *adapter) +{ + struct e1000_hw *hw = &adapter->hw; + u32 rctl, rxdctl; + + rctl = E1000_READ_REG(hw, E1000_RCTL); + E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); + E1000_WRITE_FLUSH(hw); + usec_delay(150); + + rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0)); + /* zero the lower 14 bits (prefetch and host thresholds) */ + rxdctl &= 0xffffc000; + /* + * update thresholds: prefetch threshold to 31, host threshold to 1 + * and make sure the granularity is "descriptors" and not "cache lines" + */ + rxdctl |= (0x1F | (1 << 8) | E1000_RXDCTL_THRESH_UNIT_DESC); + E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl); + + /* momentarily enable the RX ring for the changes to take effect */ + E1000_WRITE_REG(hw, E1000_RCTL, rctl | E1000_RCTL_EN); + E1000_WRITE_FLUSH(hw); + usec_delay(150); + E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); +} + +/* +** em_flush_desc_rings - remove all descriptors from the descriptor rings +** +** In i219, the descriptor rings must be emptied before resetting the HW +** or before changing the device state to D3 during runtime (runtime PM). +** +** Failure to do this will cause the HW to enter a unit hang state which can +** only be released by PCI reset on the device +** +*/ +static void +em_flush_desc_rings(struct adapter *adapter) +{ + struct e1000_hw *hw = &adapter->hw; + device_t dev = adapter->dev; + u16 hang_state; + u32 fext_nvm11, tdlen; + + /* First, disable MULR fix in FEXTNVM11 */ + fext_nvm11 = E1000_READ_REG(hw, E1000_FEXTNVM11); + fext_nvm11 |= E1000_FEXTNVM11_DISABLE_MULR_FIX; + E1000_WRITE_REG(hw, E1000_FEXTNVM11, fext_nvm11); + + /* do nothing if we're not in faulty state, or if the queue is empty */ + tdlen = E1000_READ_REG(hw, E1000_TDLEN(0)); + hang_state = pci_read_config(dev, PCICFG_DESC_RING_STATUS, 2); + if (!(hang_state & FLUSH_DESC_REQUIRED) || !tdlen) + return; + em_flush_tx_ring(adapter); + + /* recheck, maybe the fault is caused by the rx ring */ + hang_state = pci_read_config(dev, PCICFG_DESC_RING_STATUS, 2); + if (hang_state & FLUSH_DESC_REQUIRED) + em_flush_rx_ring(adapter); +} + + /********************************************************************* * * Initialize the hardware to a configuration @@ -2821,7 +3023,7 @@ static void em_reset(struct adapter *adapter) { device_t dev = adapter->dev; - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; struct e1000_hw *hw = &adapter->hw; u16 rx_buffer_size; u32 pba; @@ -2872,6 +3074,7 @@ em_reset(struct adapter *adapter) case e1000_pchlan: case e1000_pch2lan: case e1000_pch_lpt: + case e1000_pch_spt: pba = E1000_PBA_26K; break; default: @@ -2889,7 +3092,7 @@ em_reset(struct adapter *adapter) * received after sending an XOFF. * - Low water mark works best when it is very near the high water mark. * This allows the receiver to restart by sending XON when it has - * drained a bit. Here we use an arbitary value of 1500 which will + * drained a bit. Here we use an arbitrary value of 1500 which will * restart after one full frame is pulled from the buffer. There * could be several smaller frames in the buffer and if so they will * not trigger the XON until their total number reduces the buffer @@ -2919,7 +3122,7 @@ em_reset(struct adapter *adapter) /* Workaround: no TX flow ctrl for PCH */ hw->fc.requested_mode = e1000_fc_rx_pause; hw->fc.pause_time = 0xFFFF; /* override */ - if (ifp->if_mtu > ETHERMTU) { + if (if_getmtu(ifp) > ETHERMTU) { hw->fc.high_water = 0x3500; hw->fc.low_water = 0x1500; } else { @@ -2930,19 +3133,20 @@ em_reset(struct adapter *adapter) break; case e1000_pch2lan: case e1000_pch_lpt: + case e1000_pch_spt: hw->fc.high_water = 0x5C20; hw->fc.low_water = 0x5048; hw->fc.pause_time = 0x0650; hw->fc.refresh_time = 0x0400; /* Jumbos need adjusted PBA */ - if (ifp->if_mtu > ETHERMTU) + if (if_getmtu(ifp) > ETHERMTU) E1000_WRITE_REG(hw, E1000_PBA, 12); else E1000_WRITE_REG(hw, E1000_PBA, 26); break; case e1000_ich9lan: case e1000_ich10lan: - if (ifp->if_mtu > ETHERMTU) { + if (if_getmtu(ifp) > ETHERMTU) { hw->fc.high_water = 0x2800; hw->fc.low_water = hw->fc.high_water - 8; break; @@ -2954,6 +3158,10 @@ em_reset(struct adapter *adapter) break; } + /* I219 needs some special flushing to avoid hangs */ + if (hw->mac.type == e1000_pch_spt) + em_flush_desc_rings(adapter); + /* Issue a global reset */ e1000_reset_hw(hw); E1000_WRITE_REG(hw, E1000_WUC, 0); @@ -2978,47 +3186,55 @@ em_reset(struct adapter *adapter) static int em_setup_interface(device_t dev, struct adapter *adapter) { - struct ifnet *ifp; + if_t ifp; INIT_DEBUGOUT("em_setup_interface: begin"); - ifp = adapter->ifp = if_alloc(IFT_ETHER); - if (ifp == NULL) { + ifp = adapter->ifp = if_gethandle(IFT_ETHER); + if (ifp == 0) { device_printf(dev, "can not allocate ifnet structure\n"); return (-1); } if_initname(ifp, device_get_name(dev), device_get_unit(dev)); - ifp->if_init = em_init; - ifp->if_softc = adapter; - ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; - ifp->if_ioctl = em_ioctl; + if_setdev(ifp, dev); + if_setinitfn(ifp, em_init); + if_setsoftc(ifp, adapter); + if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); + if_setioctlfn(ifp, em_ioctl); + if_setgetcounterfn(ifp, em_get_counter); + + /* TSO parameters */ + ifp->if_hw_tsomax = IP_MAXPACKET; + /* Take m_pullup(9)'s in em_xmit() w/ TSO into acount. */ + ifp->if_hw_tsomaxsegcount = EM_MAX_SCATTER - 5; + ifp->if_hw_tsomaxsegsize = EM_TSO_SEG_SIZE; + #ifdef EM_MULTIQUEUE /* Multiqueue stack interface */ - ifp->if_transmit = em_mq_start; - ifp->if_qflush = em_qflush; + if_settransmitfn(ifp, em_mq_start); + if_setqflushfn(ifp, em_qflush); #else - ifp->if_start = em_start; - IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1); - ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1; - IFQ_SET_READY(&ifp->if_snd); + if_setstartfn(ifp, em_start); + if_setsendqlen(ifp, adapter->num_tx_desc - 1); + if_setsendqready(ifp); #endif ether_ifattach(ifp, adapter->hw.mac.addr); - ifp->if_capabilities = ifp->if_capenable = 0; + if_setcapabilities(ifp, 0); + if_setcapenable(ifp, 0); - ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; - ifp->if_capabilities |= IFCAP_TSO4; + if_setcapabilitiesbit(ifp, IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | + IFCAP_TSO4, 0); /* * Tell the upper layer(s) we * support full VLAN capability */ - ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); - ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING - | IFCAP_VLAN_HWTSO - | IFCAP_VLAN_MTU; - ifp->if_capenable = ifp->if_capabilities; + if_setifheaderlen(ifp, sizeof(struct ether_vlan_header)); + if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWTSO | + IFCAP_VLAN_MTU, 0); + if_setcapenable(ifp, if_getcapabilities(ifp)); /* ** Don't turn this on by default, if vlans are @@ -3028,16 +3244,16 @@ em_setup_interface(device_t dev, struct adapter *adapter) ** using vlans directly on the em driver you can ** enable this and get full hardware tag filtering. */ - ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; + if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWFILTER,0); #ifdef DEVICE_POLLING - ifp->if_capabilities |= IFCAP_POLLING; + if_setcapabilitiesbit(ifp, IFCAP_POLLING,0); #endif /* Enable only WOL MAGIC by default */ if (adapter->wol) { - ifp->if_capabilities |= IFCAP_WOL; - ifp->if_capenable |= IFCAP_WOL_MAGIC; + if_setcapabilitiesbit(ifp, IFCAP_WOL, 0); + if_setcapenablebit(ifp, IFCAP_WOL_MAGIC, 0); } /* @@ -3137,7 +3353,6 @@ fail_2: bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); bus_dma_tag_destroy(dma->dma_tag); fail_0: - dma->dma_map = NULL; dma->dma_tag = NULL; return (error); @@ -3148,12 +3363,15 @@ em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma) { if (dma->dma_tag == NULL) return; - if (dma->dma_map != NULL) { + if (dma->dma_paddr != 0) { bus_dmamap_sync(dma->dma_tag, dma->dma_map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(dma->dma_tag, dma->dma_map); + dma->dma_paddr = 0; + } + if (dma->dma_vaddr != NULL) { bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); - dma->dma_map = NULL; + dma->dma_vaddr = NULL; } bus_dma_tag_destroy(dma->dma_tag); dma->dma_tag = NULL; @@ -3239,7 +3457,7 @@ em_allocate_queues(struct adapter *adapter) * Next the RX queues... */ rsize = roundup2(adapter->num_rx_desc * - sizeof(struct e1000_rx_desc), EM_DBA_ALIGN); + sizeof(union e1000_rx_desc_extended), EM_DBA_ALIGN); for (int i = 0; i < adapter->num_queues; i++, rxconf++) { rxr = &adapter->rx_rings[i]; rxr->adapter = adapter; @@ -3257,7 +3475,7 @@ em_allocate_queues(struct adapter *adapter) error = ENOMEM; goto err_rx_desc; } - rxr->rx_base = (struct e1000_rx_desc *)rxr->rxdma.dma_vaddr; + rxr->rx_base = (union e1000_rx_desc_extended *)rxr->rxdma.dma_vaddr; bzero((void *)rxr->rx_base, rsize); /* Allocate receive buffers for the ring*/ @@ -3300,7 +3518,7 @@ em_allocate_transmit_buffers(struct tx_ring *txr) { struct adapter *adapter = txr->adapter; device_t dev = adapter->dev; - struct em_buffer *txbuf; + struct em_txbuffer *txbuf; int error, i; /* @@ -3323,7 +3541,7 @@ em_allocate_transmit_buffers(struct tx_ring *txr) } if (!(txr->tx_buffers = - (struct em_buffer *) malloc(sizeof(struct em_buffer) * + (struct em_txbuffer *) malloc(sizeof(struct em_txbuffer) * adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { device_printf(dev, "Unable to allocate tx_buffer memory\n"); error = ENOMEM; @@ -3356,11 +3574,11 @@ static void em_setup_transmit_ring(struct tx_ring *txr) { struct adapter *adapter = txr->adapter; - struct em_buffer *txbuf; + struct em_txbuffer *txbuf; int i; #ifdef DEV_NETMAP - struct netmap_adapter *na = NA(adapter->ifp); struct netmap_slot *slot; + struct netmap_adapter *na = netmap_getna(adapter->ifp); #endif /* DEV_NETMAP */ /* Clear the old descriptor contents */ @@ -3391,10 +3609,10 @@ em_setup_transmit_ring(struct tx_ring *txr) uint64_t paddr; void *addr; - addr = PNMB(slot + si, &paddr); + addr = PNMB(na, slot + si, &paddr); txr->tx_base[i].buffer_addr = htole64(paddr); /* reload the map for netmap mode */ - netmap_load_map(txr->txtag, txbuf->map, addr); + netmap_load_map(na, txr->txtag, txbuf->map, addr); } #endif /* DEV_NETMAP */ @@ -3404,7 +3622,7 @@ em_setup_transmit_ring(struct tx_ring *txr) /* Set number of descriptors available */ txr->tx_avail = adapter->num_tx_desc; - txr->queue_status = EM_QUEUE_IDLE; + txr->busy = EM_TX_IDLE; /* Clear checksum offload context. */ txr->last_hw_offload = 0; @@ -3444,7 +3662,7 @@ em_initialize_transmit_unit(struct adapter *adapter) { struct tx_ring *txr = adapter->tx_rings; struct e1000_hw *hw = &adapter->hw; - u32 tctl, tarc, tipg = 0; + u32 tctl, txdctl = 0, tarc, tipg = 0; INIT_DEBUGOUT("em_initialize_transmit_unit: begin"); @@ -3465,7 +3683,16 @@ em_initialize_transmit_unit(struct adapter *adapter) E1000_READ_REG(&adapter->hw, E1000_TDBAL(i)), E1000_READ_REG(&adapter->hw, E1000_TDLEN(i))); - txr->queue_status = EM_QUEUE_IDLE; + txr->busy = EM_TX_IDLE; + txdctl = 0; /* clear txdctl */ + txdctl |= 0x1f; /* PTHRESH */ + txdctl |= 1 << 8; /* HTHRESH */ + txdctl |= 1 << 16;/* WTHRESH */ + txdctl |= 1 << 22; /* Reserved bit 22 must always be 1 */ + txdctl |= E1000_TXDCTL_GRAN; + txdctl |= 1 << 25; /* LWTHRESH */ + + E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl); } /* Set the default values for the Tx Inter Packet Gap timer */ @@ -3496,15 +3723,25 @@ em_initialize_transmit_unit(struct adapter *adapter) if ((adapter->hw.mac.type == e1000_82571) || (adapter->hw.mac.type == e1000_82572)) { tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); - tarc |= SPEED_MODE_BIT; + tarc |= TARC_SPEED_MODE_BIT; E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); } else if (adapter->hw.mac.type == e1000_80003es2lan) { + /* errata: program both queues to unweighted RR */ tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); tarc |= 1; E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1)); tarc |= 1; E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc); + } else if (adapter->hw.mac.type == e1000_82574) { + tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); + tarc |= TARC_ERRATA_BIT; + if ( adapter->num_queues > 1) { + tarc |= (TARC_COMPENSATION_MODE | TARC_MQ_FIX); + E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); + E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc); + } else + E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); } adapter->txd_cmd = E1000_TXD_CMD_IFCS; @@ -3523,6 +3760,15 @@ em_initialize_transmit_unit(struct adapter *adapter) /* This write will effectively turn on the transmit unit. */ E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); + if (hw->mac.type == e1000_pch_spt) { + u32 reg; + reg = E1000_READ_REG(hw, E1000_IOSFPC); + reg |= E1000_RCTL_RDMTS_HEX; + E1000_WRITE_REG(hw, E1000_IOSFPC, reg); + reg = E1000_READ_REG(hw, E1000_TARC(0)); + reg |= E1000_TARC0_CB_MULTIQ_3_REQ; + E1000_WRITE_REG(hw, E1000_TARC(0), reg); + } } @@ -3556,7 +3802,7 @@ static void em_free_transmit_buffers(struct tx_ring *txr) { struct adapter *adapter = txr->adapter; - struct em_buffer *txbuf; + struct em_txbuffer *txbuf; INIT_DEBUGOUT("free_transmit_ring: begin"); @@ -3623,7 +3869,7 @@ em_transmit_checksum_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off, { struct adapter *adapter = txr->adapter; struct e1000_context_desc *TXD = NULL; - struct em_buffer *tx_buffer; + struct em_txbuffer *tx_buffer; int cur, hdr_len; u32 cmd = 0; u16 offload = 0; @@ -3657,29 +3903,38 @@ em_transmit_checksum_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off, offload |= CSUM_TCP; tucss = hdr_len; tucso = hdr_len + offsetof(struct tcphdr, th_sum); - /* - * Setting up new checksum offload context for every frames - * takes a lot of processing time for hardware. This also - * reduces performance a lot for small sized frames so avoid - * it if driver can use previously configured checksum - * offload context. - */ - if (txr->last_hw_offload == offload) { - if (offload & CSUM_IP) { - if (txr->last_hw_ipcss == ipcss && - txr->last_hw_ipcso == ipcso && - txr->last_hw_tucss == tucss && - txr->last_hw_tucso == tucso) - return; - } else { - if (txr->last_hw_tucss == tucss && - txr->last_hw_tucso == tucso) - return; - } - } - txr->last_hw_offload = offload; - txr->last_hw_tucss = tucss; - txr->last_hw_tucso = tucso; + /* + * The 82574L can only remember the *last* context used + * regardless of queue that it was use for. We cannot reuse + * contexts on this hardware platform and must generate a new + * context every time. 82574L hardware spec, section 7.2.6, + * second note. + */ + if (adapter->num_queues < 2) { + /* + * Setting up new checksum offload context for every + * frames takes a lot of processing time for hardware. + * This also reduces performance a lot for small sized + * frames so avoid it if driver can use previously + * configured checksum offload context. + */ + if (txr->last_hw_offload == offload) { + if (offload & CSUM_IP) { + if (txr->last_hw_ipcss == ipcss && + txr->last_hw_ipcso == ipcso && + txr->last_hw_tucss == tucss && + txr->last_hw_tucso == tucso) + return; + } else { + if (txr->last_hw_tucss == tucss && + txr->last_hw_tucso == tucso) + return; + } + } + txr->last_hw_offload = offload; + txr->last_hw_tucss = tucss; + txr->last_hw_tucso = tucso; + } /* * Start offset for payload checksum calculation. * End offset for payload checksum calculation. @@ -3695,29 +3950,38 @@ em_transmit_checksum_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off, *txd_upper |= E1000_TXD_POPTS_TXSM << 8; tucss = hdr_len; tucso = hdr_len + offsetof(struct udphdr, uh_sum); - /* - * Setting up new checksum offload context for every frames - * takes a lot of processing time for hardware. This also - * reduces performance a lot for small sized frames so avoid - * it if driver can use previously configured checksum - * offload context. - */ - if (txr->last_hw_offload == offload) { - if (offload & CSUM_IP) { - if (txr->last_hw_ipcss == ipcss && - txr->last_hw_ipcso == ipcso && - txr->last_hw_tucss == tucss && - txr->last_hw_tucso == tucso) - return; - } else { - if (txr->last_hw_tucss == tucss && - txr->last_hw_tucso == tucso) - return; + /* + * The 82574L can only remember the *last* context used + * regardless of queue that it was use for. We cannot reuse + * contexts on this hardware platform and must generate a new + * context every time. 82574L hardware spec, section 7.2.6, + * second note. + */ + if (adapter->num_queues < 2) { + /* + * Setting up new checksum offload context for every + * frames takes a lot of processing time for hardware. + * This also reduces performance a lot for small sized + * frames so avoid it if driver can use previously + * configured checksum offload context. + */ + if (txr->last_hw_offload == offload) { + if (offload & CSUM_IP) { + if (txr->last_hw_ipcss == ipcss && + txr->last_hw_ipcso == ipcso && + txr->last_hw_tucss == tucss && + txr->last_hw_tucso == tucso) + return; + } else { + if (txr->last_hw_tucss == tucss && + txr->last_hw_tucso == tucso) + return; + } } - } - txr->last_hw_offload = offload; - txr->last_hw_tucss = tucss; - txr->last_hw_tucso = tucso; + txr->last_hw_offload = offload; + txr->last_hw_tucss = tucss; + txr->last_hw_tucso = tucso; + } /* * Start offset for header checksum calculation. * End offset for header checksum calculation. @@ -3760,7 +4024,7 @@ em_tso_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off, { struct adapter *adapter = txr->adapter; struct e1000_context_desc *TXD; - struct em_buffer *tx_buffer; + struct em_txbuffer *tx_buffer; int cur, hdr_len; /* @@ -3838,9 +4102,9 @@ em_txeof(struct tx_ring *txr) { struct adapter *adapter = txr->adapter; int first, last, done, processed; - struct em_buffer *tx_buffer; + struct em_txbuffer *tx_buffer; struct e1000_tx_desc *tx_desc, *eop_desc; - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; EM_TX_LOCK_ASSERT(txr); #ifdef DEV_NETMAP @@ -3848,9 +4112,9 @@ em_txeof(struct tx_ring *txr) return; #endif /* DEV_NETMAP */ - /* No work, make sure watchdog is off */ + /* No work, make sure hang detection is disabled */ if (txr->tx_avail == adapter->num_tx_desc) { - txr->queue_status = EM_QUEUE_IDLE; + txr->busy = EM_TX_IDLE; return; } @@ -3893,7 +4157,6 @@ em_txeof(struct tx_ring *txr) tx_buffer->m_head = NULL; } tx_buffer->next_eop = -1; - txr->watchdog_time = ticks; if (++first == adapter->num_tx_desc) first = 0; @@ -3901,7 +4164,7 @@ em_txeof(struct tx_ring *txr) tx_buffer = &txr->tx_buffers[first]; tx_desc = &txr->tx_base[first]; } - ++ifp->if_opackets; + if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); /* See if we can continue to the next packet */ last = tx_buffer->next_eop; if (last != -1) { @@ -3918,14 +4181,16 @@ em_txeof(struct tx_ring *txr) txr->next_to_clean = first; /* - ** Watchdog calculation, we know there's - ** work outstanding or the first return - ** would have been taken, so none processed - ** for too long indicates a hang. local timer - ** will examine this and do a reset if needed. + ** Hang detection: we know there's work outstanding + ** or the entry return would have been taken, so no + ** descriptor processed here indicates a potential hang. + ** The local timer will examine this and do a reset if needed. */ - if ((!processed) && ((ticks - txr->watchdog_time) > EM_WATCHDOG)) - txr->queue_status = EM_QUEUE_HUNG; + if (processed == 0) { + if (txr->busy != EM_TX_HUNG) + ++txr->busy; + } else /* At least one descriptor was cleaned */ + txr->busy = EM_TX_BUSY; /* note this clears HUNG */ /* * If we have a minimum free, clear IFF_DRV_OACTIVE @@ -3934,16 +4199,15 @@ em_txeof(struct tx_ring *txr) * TX lock which, with a single queue, guarantees * sanity. */ - if (txr->tx_avail >= EM_MAX_SCATTER) - ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; + if (txr->tx_avail >= EM_MAX_SCATTER) { + if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); + } - /* Disable watchdog if all clean */ - if (txr->tx_avail == adapter->num_tx_desc) { - txr->queue_status = EM_QUEUE_IDLE; - } + /* Disable hang detection if all clean */ + if (txr->tx_avail == adapter->num_tx_desc) + txr->busy = EM_TX_IDLE; } - /********************************************************************* * * Refresh RX descriptor mbufs from system mbuf buffer pool. @@ -3954,8 +4218,8 @@ em_refresh_mbufs(struct rx_ring *rxr, int limit) { struct adapter *adapter = rxr->adapter; struct mbuf *m; - bus_dma_segment_t segs[1]; - struct em_buffer *rxbuf; + bus_dma_segment_t segs; + struct em_rxbuffer *rxbuf; int i, j, error, nsegs; bool cleaned = FALSE; @@ -3990,7 +4254,7 @@ em_refresh_mbufs(struct rx_ring *rxr, int limit) /* Use bus_dma machinery to setup the memory mapping */ error = bus_dmamap_load_mbuf_sg(rxr->rxtag, rxbuf->map, - m, segs, &nsegs, BUS_DMA_NOWAIT); + m, &segs, &nsegs, BUS_DMA_NOWAIT); if (error != 0) { printf("Refresh mbufs: hdr dmamap load" " failure - %d\n", error); @@ -3999,9 +4263,10 @@ em_refresh_mbufs(struct rx_ring *rxr, int limit) goto update; } rxbuf->m_head = m; + rxbuf->paddr = segs.ds_addr; bus_dmamap_sync(rxr->rxtag, rxbuf->map, BUS_DMASYNC_PREREAD); - rxr->rx_base[i].buffer_addr = htole64(segs[0].ds_addr); + em_setup_rxdesc(&rxr->rx_base[i], rxbuf); cleaned = TRUE; i = j; /* Next is precalulated for us */ @@ -4036,10 +4301,10 @@ em_allocate_receive_buffers(struct rx_ring *rxr) { struct adapter *adapter = rxr->adapter; device_t dev = adapter->dev; - struct em_buffer *rxbuf; + struct em_rxbuffer *rxbuf; int error; - rxr->rx_buffers = malloc(sizeof(struct em_buffer) * + rxr->rx_buffers = malloc(sizeof(struct em_rxbuffer) * adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); if (rxr->rx_buffers == NULL) { device_printf(dev, "Unable to allocate rx_buffer memory\n"); @@ -4092,22 +4357,22 @@ static int em_setup_receive_ring(struct rx_ring *rxr) { struct adapter *adapter = rxr->adapter; - struct em_buffer *rxbuf; + struct em_rxbuffer *rxbuf; bus_dma_segment_t seg[1]; int rsize, nsegs, error = 0; #ifdef DEV_NETMAP - struct netmap_adapter *na = NA(adapter->ifp); struct netmap_slot *slot; + struct netmap_adapter *na = netmap_getna(adapter->ifp); #endif /* Clear the ring contents */ EM_RX_LOCK(rxr); rsize = roundup2(adapter->num_rx_desc * - sizeof(struct e1000_rx_desc), EM_DBA_ALIGN); + sizeof(union e1000_rx_desc_extended), EM_DBA_ALIGN); bzero((void *)rxr->rx_base, rsize); #ifdef DEV_NETMAP - slot = netmap_reset(na, NR_RX, 0, 0); + slot = netmap_reset(na, NR_RX, rxr->me, 0); #endif /* @@ -4133,10 +4398,10 @@ em_setup_receive_ring(struct rx_ring *rxr) uint64_t paddr; void *addr; - addr = PNMB(slot + si, &paddr); - netmap_load_map(rxr->rxtag, rxbuf->map, addr); - /* Update descriptor */ - rxr->rx_base[j].buffer_addr = htole64(paddr); + addr = PNMB(na, slot + si, &paddr); + netmap_load_map(na, rxr->rxtag, rxbuf->map, addr); + rxbuf->paddr = paddr; + em_setup_rxdesc(&rxr->rx_base[j], rxbuf); continue; } #endif /* DEV_NETMAP */ @@ -4162,8 +4427,8 @@ em_setup_receive_ring(struct rx_ring *rxr) bus_dmamap_sync(rxr->rxtag, rxbuf->map, BUS_DMASYNC_PREREAD); - /* Update descriptor */ - rxr->rx_base[j].buffer_addr = htole64(seg[0].ds_addr); + rxbuf->paddr = seg[0].ds_addr; + em_setup_rxdesc(&rxr->rx_base[j], rxbuf); } rxr->next_to_check = 0; rxr->next_to_refresh = 0; @@ -4200,7 +4465,7 @@ fail: for (int i = 0; i < q; ++i) { rxr = &adapter->rx_rings[i]; for (int n = 0; n < adapter->num_rx_desc; n++) { - struct em_buffer *rxbuf; + struct em_rxbuffer *rxbuf; rxbuf = &rxr->rx_buffers[n]; if (rxbuf->m_head != NULL) { bus_dmamap_sync(rxr->rxtag, rxbuf->map, @@ -4247,7 +4512,7 @@ static void em_free_receive_buffers(struct rx_ring *rxr) { struct adapter *adapter = rxr->adapter; - struct em_buffer *rxbuf = NULL; + struct em_rxbuffer *rxbuf = NULL; INIT_DEBUGOUT("free_receive_buffers: begin"); @@ -4289,11 +4554,10 @@ em_free_receive_buffers(struct rx_ring *rxr) static void em_initialize_receive_unit(struct adapter *adapter) { - struct rx_ring *rxr = adapter->rx_rings; - struct ifnet *ifp = adapter->ifp; + struct rx_ring *rxr = adapter->rx_rings; + if_t ifp = adapter->ifp; struct e1000_hw *hw = &adapter->hw; - u64 bus_addr; - u32 rctl, rxcsum; + u32 rctl, rxcsum, rfctl; INIT_DEBUGOUT("em_initialize_receive_units: begin"); @@ -4306,14 +4570,39 @@ em_initialize_receive_unit(struct adapter *adapter) if ((hw->mac.type != e1000_82574) && (hw->mac.type != e1000_82583)) E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); + /* Setup the Receive Control Register */ + rctl &= ~(3 << E1000_RCTL_MO_SHIFT); + rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | + E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF | + (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT); + + /* Do not store bad packets */ + rctl &= ~E1000_RCTL_SBP; + + /* Enable Long Packet receive */ + if (if_getmtu(ifp) > ETHERMTU) + rctl |= E1000_RCTL_LPE; + else + rctl &= ~E1000_RCTL_LPE; + + /* Strip the CRC */ + if (!em_disable_crc_stripping) + rctl |= E1000_RCTL_SECRC; + E1000_WRITE_REG(&adapter->hw, E1000_RADV, adapter->rx_abs_int_delay.value); + + E1000_WRITE_REG(&adapter->hw, E1000_RDTR, + adapter->rx_int_delay.value); /* * Set the interrupt throttling rate. Value is calculated * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) */ E1000_WRITE_REG(hw, E1000_ITR, DEFAULT_ITR); + /* Use extended rx descriptor formats */ + rfctl = E1000_READ_REG(hw, E1000_RFCTL); + rfctl |= E1000_RFCTL_EXTEN; /* ** When using MSIX interrupts we need to throttle ** using the EITR register (82574 only) @@ -4323,16 +4612,65 @@ em_initialize_receive_unit(struct adapter *adapter) E1000_WRITE_REG(hw, E1000_EITR_82574(i), DEFAULT_ITR); /* Disable accelerated acknowledge */ - E1000_WRITE_REG(hw, E1000_RFCTL, E1000_RFCTL_ACK_DIS); + rfctl |= E1000_RFCTL_ACK_DIS; } + E1000_WRITE_REG(hw, E1000_RFCTL, rfctl); rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); - if (ifp->if_capenable & IFCAP_RXCSUM) + if (if_getcapenable(ifp) & IFCAP_RXCSUM) { +#ifdef EM_MULTIQUEUE + rxcsum |= E1000_RXCSUM_TUOFL | + E1000_RXCSUM_IPOFL | + E1000_RXCSUM_PCSD; +#else rxcsum |= E1000_RXCSUM_TUOFL; - else +#endif + } else rxcsum &= ~E1000_RXCSUM_TUOFL; + E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); +#ifdef EM_MULTIQUEUE +#define RSSKEYLEN 10 + if (adapter->num_queues > 1) { + uint8_t rss_key[4 * RSSKEYLEN]; + uint32_t reta = 0; + int i; + + /* + * Configure RSS key + */ + arc4rand(rss_key, sizeof(rss_key), 0); + for (i = 0; i < RSSKEYLEN; ++i) { + uint32_t rssrk = 0; + + rssrk = EM_RSSRK_VAL(rss_key, i); + E1000_WRITE_REG(hw,E1000_RSSRK(i), rssrk); + } + + /* + * Configure RSS redirect table in following fashion: + * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)] + */ + for (i = 0; i < sizeof(reta); ++i) { + uint32_t q; + + q = (i % adapter->num_queues) << 7; + reta |= q << (8 * i); + } + + for (i = 0; i < 32; ++i) { + E1000_WRITE_REG(hw, E1000_RETA(i), reta); + } + + E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_RSS_ENABLE_2Q | + E1000_MRQC_RSS_FIELD_IPV4_TCP | + E1000_MRQC_RSS_FIELD_IPV4 | + E1000_MRQC_RSS_FIELD_IPV6_TCP_EX | + E1000_MRQC_RSS_FIELD_IPV6_EX | + E1000_MRQC_RSS_FIELD_IPV6); + } +#endif /* ** XXX TEMPORARY WORKAROUND: on some systems with 82573 ** long latencies are observed, like Lenovo X60. This @@ -4345,11 +4683,11 @@ em_initialize_receive_unit(struct adapter *adapter) for (int i = 0; i < adapter->num_queues; i++, rxr++) { /* Setup the Base and Length of the Rx Descriptor Ring */ + u64 bus_addr = rxr->rxdma.dma_paddr; u32 rdt = adapter->num_rx_desc - 1; /* default */ - bus_addr = rxr->rxdma.dma_paddr; E1000_WRITE_REG(hw, E1000_RDLEN(i), - adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); + adapter->num_rx_desc * sizeof(union e1000_rx_desc_extended)); E1000_WRITE_REG(hw, E1000_RDBAH(i), (u32)(bus_addr >> 32)); E1000_WRITE_REG(hw, E1000_RDBAL(i), (u32)bus_addr); /* Setup the Head and Tail Descriptor Pointers */ @@ -4359,40 +4697,48 @@ em_initialize_receive_unit(struct adapter *adapter) * an init() while a netmap client is active must * preserve the rx buffers passed to userspace. */ - if (ifp->if_capenable & IFCAP_NETMAP) - rdt -= nm_kr_rxspace(&NA(adapter->ifp)->rx_rings[i]); + if (if_getcapenable(ifp) & IFCAP_NETMAP) { + struct netmap_adapter *na = netmap_getna(adapter->ifp); + rdt -= nm_kr_rxspace(&na->rx_rings[i]); + } #endif /* DEV_NETMAP */ E1000_WRITE_REG(hw, E1000_RDT(i), rdt); } - /* Set PTHRESH for improved jumbo performance */ + /* + * Set PTHRESH for improved jumbo performance + * According to 10.2.5.11 of Intel 82574 Datasheet, + * RXDCTL(1) is written whenever RXDCTL(0) is written. + * Only write to RXDCTL(1) if there is a need for different + * settings. + */ if (((adapter->hw.mac.type == e1000_ich9lan) || (adapter->hw.mac.type == e1000_pch2lan) || (adapter->hw.mac.type == e1000_ich10lan)) && - (ifp->if_mtu > ETHERMTU)) { + (if_getmtu(ifp) > ETHERMTU)) { u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0)); E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl | 3); + } else if (adapter->hw.mac.type == e1000_82574) { + for (int i = 0; i < adapter->num_queues; i++) { + u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); + + rxdctl |= 0x20; /* PTHRESH */ + rxdctl |= 4 << 8; /* HTHRESH */ + rxdctl |= 4 << 16;/* WTHRESH */ + rxdctl |= 1 << 24; /* Switch to granularity */ + E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); + } } if (adapter->hw.mac.type >= e1000_pch2lan) { - if (ifp->if_mtu > ETHERMTU) + if (if_getmtu(ifp) > ETHERMTU) e1000_lv_jumbo_workaround_ich8lan(hw, TRUE); else e1000_lv_jumbo_workaround_ich8lan(hw, FALSE); } - /* Setup the Receive Control Register */ - rctl &= ~(3 << E1000_RCTL_MO_SHIFT); - rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | - E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF | - (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT); - - /* Strip the CRC */ - rctl |= E1000_RCTL_SECRC; - /* Make sure VLAN Filters are off */ rctl &= ~E1000_RCTL_VFE; - rctl &= ~E1000_RCTL_SBP; if (adapter->rx_mbuf_sz == MCLBYTES) rctl |= E1000_RCTL_SZ_2048; @@ -4401,11 +4747,8 @@ em_initialize_receive_unit(struct adapter *adapter) else if (adapter->rx_mbuf_sz > MJUMPAGESIZE) rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX; - if (ifp->if_mtu > ETHERMTU) - rctl |= E1000_RCTL_LPE; - else - rctl &= ~E1000_RCTL_LPE; - + /* ensure we clear use DTYPE of 00 here */ + rctl &= ~0x00000C00; /* Write out the settings */ E1000_WRITE_REG(hw, E1000_RCTL, rctl); @@ -4428,16 +4771,21 @@ static bool em_rxeof(struct rx_ring *rxr, int count, int *done) { struct adapter *adapter = rxr->adapter; - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; struct mbuf *mp, *sendmp; - u8 status = 0; + u32 status = 0; u16 len; int i, processed, rxdone = 0; bool eop; - struct e1000_rx_desc *cur; + union e1000_rx_desc_extended *cur; EM_RX_LOCK(rxr); + /* Sync the ring */ + bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + + #ifdef DEV_NETMAP if (netmap_rx_irq(ifp, rxr->me, &processed)) { EM_RX_UNLOCK(rxr); @@ -4446,24 +4794,20 @@ em_rxeof(struct rx_ring *rxr, int count, int *done) #endif /* DEV_NETMAP */ for (i = rxr->next_to_check, processed = 0; count != 0;) { - - if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) + if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) break; - bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - cur = &rxr->rx_base[i]; - status = cur->status; + status = le32toh(cur->wb.upper.status_error); mp = sendmp = NULL; if ((status & E1000_RXD_STAT_DD) == 0) break; - len = le16toh(cur->length); + len = le16toh(cur->wb.upper.length); eop = (status & E1000_RXD_STAT_EOP) != 0; - if ((cur->errors & E1000_RXD_ERR_FRAME_ERR_MASK) || + if ((status & E1000_RXDEXT_ERR_FRAME_ERR_MASK) || (rxr->discard == TRUE)) { adapter->dropped_pkts++; ++rxr->rx_discarded; @@ -4498,9 +4842,9 @@ em_rxeof(struct rx_ring *rxr, int count, int *done) if (eop) { --count; sendmp = rxr->fmp; - sendmp->m_pkthdr.rcvif = ifp; - ifp->if_ipackets++; - em_receive_checksum(cur, sendmp); + if_setrcvif(sendmp, ifp); + if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); + em_receive_checksum(status, sendmp); #ifndef __NO_STRICT_ALIGNMENT if (adapter->hw.mac.max_frame_size > (MCLBYTES - ETHER_ALIGN) && @@ -4508,8 +4852,8 @@ em_rxeof(struct rx_ring *rxr, int count, int *done) goto skip; #endif if (status & E1000_RXD_STAT_VP) { - sendmp->m_pkthdr.ether_vtag = - le16toh(cur->special); + if_setvtag(sendmp, + le16toh(cur->wb.upper.vlan)); sendmp->m_flags |= M_VLANTAG; } #ifndef __NO_STRICT_ALIGNMENT @@ -4518,8 +4862,12 @@ skip: rxr->fmp = rxr->lmp = NULL; } next_desc: + /* Sync the ring */ + bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + /* Zero out the receive descriptors status. */ - cur->status = 0; + cur->wb.upper.status_error &= htole32(~0xFF); ++rxdone; /* cumulative for POLL */ ++processed; @@ -4531,7 +4879,7 @@ next_desc: if (sendmp != NULL) { rxr->next_to_check = i; EM_RX_UNLOCK(rxr); - (*ifp->if_input)(ifp, sendmp); + if_input(ifp, sendmp); EM_RX_LOCK(rxr); i = rxr->next_to_check; } @@ -4558,7 +4906,7 @@ next_desc: static __inline void em_rx_discard(struct rx_ring *rxr, int i) { - struct em_buffer *rbuf; + struct em_rxbuffer *rbuf; rbuf = &rxr->rx_buffers[i]; bus_dmamap_unload(rxr->rxtag, rbuf->map); @@ -4630,6 +4978,14 @@ em_fixup_rx(struct rx_ring *rxr) } #endif +static void +em_setup_rxdesc(union e1000_rx_desc_extended *rxd, const struct em_rxbuffer *rxbuf) +{ + rxd->read.buffer_addr = htole64(rxbuf->paddr); + /* DD bits must be cleared */ + rxd->wb.upper.status_error= 0; +} + /********************************************************************* * * Verify that the hardware indicated that the checksum is valid. @@ -4638,23 +4994,27 @@ em_fixup_rx(struct rx_ring *rxr) * *********************************************************************/ static void -em_receive_checksum(struct e1000_rx_desc *rx_desc, struct mbuf *mp) +em_receive_checksum(uint32_t status, struct mbuf *mp) { mp->m_pkthdr.csum_flags = 0; /* Ignore Checksum bit is set */ - if (rx_desc->status & E1000_RXD_STAT_IXSM) + if (status & E1000_RXD_STAT_IXSM) return; - if (rx_desc->errors & (E1000_RXD_ERR_TCPE | E1000_RXD_ERR_IPE)) - return; - - /* IP Checksum Good? */ - if (rx_desc->status & E1000_RXD_STAT_IPCS) + /* If the IP checksum exists and there is no IP Checksum error */ + if ((status & (E1000_RXD_STAT_IPCS | E1000_RXDEXT_STATERR_IPE)) == + E1000_RXD_STAT_IPCS) { mp->m_pkthdr.csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID); + } /* TCP or UDP checksum */ - if (rx_desc->status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) { + if ((status & (E1000_RXD_STAT_TCPCS | E1000_RXDEXT_STATERR_TCPE)) == + E1000_RXD_STAT_TCPCS) { + mp->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); + mp->m_pkthdr.csum_data = htons(0xffff); + } + if (status & E1000_RXD_STAT_UDPCS) { mp->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); mp->m_pkthdr.csum_data = htons(0xffff); } @@ -4665,12 +5025,12 @@ em_receive_checksum(struct e1000_rx_desc *rx_desc, struct mbuf *mp) * config EVENT */ static void -em_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) +em_register_vlan(void *arg, if_t ifp, u16 vtag) { - struct adapter *adapter = ifp->if_softc; + struct adapter *adapter = if_getsoftc(ifp); u32 index, bit; - if (ifp->if_softc != arg) /* Not our event */ + if ((void*)adapter != arg) /* Not our event */ return; if ((vtag == 0) || (vtag > 4095)) /* Invalid ID */ @@ -4682,7 +5042,7 @@ em_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) adapter->shadow_vfta[index] |= (1 << bit); ++adapter->num_vlans; /* Re-init to load the changes */ - if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) + if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) em_init_locked(adapter); EM_CORE_UNLOCK(adapter); } @@ -4692,12 +5052,12 @@ em_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) * unconfig EVENT */ static void -em_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) +em_unregister_vlan(void *arg, if_t ifp, u16 vtag) { - struct adapter *adapter = ifp->if_softc; + struct adapter *adapter = if_getsoftc(ifp); u32 index, bit; - if (ifp->if_softc != arg) + if (adapter != arg) return; if ((vtag == 0) || (vtag > 4095)) /* Invalid */ @@ -4709,7 +5069,7 @@ em_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) adapter->shadow_vfta[index] &= ~(1 << bit); --adapter->num_vlans; /* Re-init to load the changes */ - if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) + if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) em_init_locked(adapter); EM_CORE_UNLOCK(adapter); } @@ -4756,8 +5116,8 @@ em_enable_intr(struct adapter *adapter) u32 ims_mask = IMS_ENABLE_MASK; if (hw->mac.type == e1000_82574) { - E1000_WRITE_REG(hw, EM_EIAC, EM_MSIX_MASK); - ims_mask |= EM_MSIX_MASK; + E1000_WRITE_REG(hw, EM_EIAC, adapter->ims); + ims_mask |= adapter->ims; } E1000_WRITE_REG(hw, E1000_IMS, ims_mask); } @@ -4963,7 +5323,7 @@ static void em_enable_wakeup(device_t dev) { struct adapter *adapter = device_get_softc(dev); - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; u32 pmc, ctrl, ctrl_ext, rctl; u16 status; @@ -4994,10 +5354,10 @@ em_enable_wakeup(device_t dev) ** Determine type of Wakeup: note that wol ** is set with all bits on by default. */ - if ((ifp->if_capenable & IFCAP_WOL_MAGIC) == 0) + if ((if_getcapenable(ifp) & IFCAP_WOL_MAGIC) == 0) adapter->wol &= ~E1000_WUFC_MAG; - if ((ifp->if_capenable & IFCAP_WOL_MCAST) == 0) + if ((if_getcapenable(ifp) & IFCAP_WOL_MCAST) == 0) adapter->wol &= ~E1000_WUFC_MC; else { rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); @@ -5020,7 +5380,7 @@ em_enable_wakeup(device_t dev) /* Request PME */ status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2); status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE); - if (ifp->if_capenable & IFCAP_WOL) + if (if_getcapenable(ifp) & IFCAP_WOL) status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2); @@ -5156,7 +5516,6 @@ em_disable_aspm(struct adapter *adapter) static void em_update_stats_counters(struct adapter *adapter) { - struct ifnet *ifp; if(adapter->hw.phy.media_type == e1000_media_type_copper || (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) { @@ -5175,12 +5534,7 @@ em_update_stats_counters(struct adapter *adapter) adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC); adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC); adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC); - /* - ** For watchdog management we need to know if we have been - ** paused during the last interval, so capture that here. - */ - adapter->pause_frames = E1000_READ_REG(&adapter->hw, E1000_XOFFRXC); - adapter->stats.xoffrxc += adapter->pause_frames; + adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC); adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC); adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC); adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64); @@ -5248,19 +5602,29 @@ em_update_stats_counters(struct adapter *adapter) adapter->stats.tsctfc += E1000_READ_REG(&adapter->hw, E1000_TSCTFC); } - ifp = adapter->ifp; - - ifp->if_collisions = adapter->stats.colc; +} - /* Rx Errors */ - ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc + - adapter->stats.crcerrs + adapter->stats.algnerrc + - adapter->stats.ruc + adapter->stats.roc + - adapter->stats.mpc + adapter->stats.cexterr; +static uint64_t +em_get_counter(if_t ifp, ift_counter cnt) +{ + struct adapter *adapter; - /* Tx Errors */ - ifp->if_oerrors = adapter->stats.ecol + - adapter->stats.latecol + adapter->watchdog_events; + adapter = if_getsoftc(ifp); + + switch (cnt) { + case IFCOUNTER_COLLISIONS: + return (adapter->stats.colc); + case IFCOUNTER_IERRORS: + return (adapter->dropped_pkts + adapter->stats.rxerrc + + adapter->stats.crcerrs + adapter->stats.algnerrc + + adapter->stats.ruc + adapter->stats.roc + + adapter->stats.mpc + adapter->stats.cexterr); + case IFCOUNTER_OERRORS: + return (adapter->stats.ecol + adapter->stats.latecol + + adapter->watchdog_events); + default: + return (if_get_counter_default(ifp, cnt)); + } } /* Export a single 32-bit register via a read-only sysctl. */ @@ -5298,18 +5662,15 @@ em_add_hw_stats(struct adapter *adapter) char namebuf[QUEUE_NAME_LEN]; /* Driver Statistics */ - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq", - CTLFLAG_RD, &adapter->link_irq, - "Link MSIX IRQ Handled"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_alloc_fail", - CTLFLAG_RD, &adapter->mbuf_alloc_failed, - "Std mbuf failed"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "cluster_alloc_fail", - CTLFLAG_RD, &adapter->mbuf_cluster_failed, - "Std mbuf cluster failed"); SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", CTLFLAG_RD, &adapter->dropped_pkts, "Driver dropped packets"); + SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq", + CTLFLAG_RD, &adapter->link_irq, + "Link MSIX IRQ Handled"); + SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_fail", + CTLFLAG_RD, &adapter->mbuf_defrag_failed, + "Defragmenting mbuf chain failed"); SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail", CTLFLAG_RD, &adapter->no_tx_dma_setup, "Driver tx dma failure in xmit"); @@ -5335,10 +5696,10 @@ em_add_hw_stats(struct adapter *adapter) CTLFLAG_RD, &adapter->hw.fc.low_water, 0, "Flow Control Low Watermark"); - for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { - snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i); + for (int i = 0; i < adapter->num_queues; i++, txr++, rxr++) { + snprintf(namebuf, QUEUE_NAME_LEN, "queue_tx_%d", i); queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, - CTLFLAG_RD, NULL, "Queue Name"); + CTLFLAG_RD, NULL, "TX Queue Name"); queue_list = SYSCTL_CHILDREN(queue_node); SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head", @@ -5357,7 +5718,12 @@ em_add_hw_stats(struct adapter *adapter) SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_desc_avail", CTLFLAG_RD, &txr->no_desc_avail, "Queue No Descriptor Available"); - + + snprintf(namebuf, QUEUE_NAME_LEN, "queue_rx_%d", i); + queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, + CTLFLAG_RD, NULL, "RX Queue Name"); + queue_list = SYSCTL_CHILDREN(queue_node); + SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head", CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDH(rxr->me), @@ -5781,29 +6147,98 @@ em_print_debug_info(struct adapter *adapter) struct tx_ring *txr = adapter->tx_rings; struct rx_ring *rxr = adapter->rx_rings; - if (adapter->ifp->if_drv_flags & IFF_DRV_RUNNING) + if (if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) printf("Interface is RUNNING "); else printf("Interface is NOT RUNNING\n"); - if (adapter->ifp->if_drv_flags & IFF_DRV_OACTIVE) + if (if_getdrvflags(adapter->ifp) & IFF_DRV_OACTIVE) printf("and INACTIVE\n"); else printf("and ACTIVE\n"); - device_printf(dev, "hw tdh = %d, hw tdt = %d\n", - E1000_READ_REG(&adapter->hw, E1000_TDH(0)), - E1000_READ_REG(&adapter->hw, E1000_TDT(0))); - device_printf(dev, "hw rdh = %d, hw rdt = %d\n", - E1000_READ_REG(&adapter->hw, E1000_RDH(0)), - E1000_READ_REG(&adapter->hw, E1000_RDT(0))); - device_printf(dev, "Tx Queue Status = %d\n", txr->queue_status); - device_printf(dev, "TX descriptors avail = %d\n", - txr->tx_avail); - device_printf(dev, "Tx Descriptors avail failure = %ld\n", - txr->no_desc_avail); - device_printf(dev, "RX discarded packets = %ld\n", - rxr->rx_discarded); - device_printf(dev, "RX Next to Check = %d\n", rxr->next_to_check); - device_printf(dev, "RX Next to Refresh = %d\n", rxr->next_to_refresh); + for (int i = 0; i < adapter->num_queues; i++, txr++, rxr++) { + device_printf(dev, "TX Queue %d ------\n", i); + device_printf(dev, "hw tdh = %d, hw tdt = %d\n", + E1000_READ_REG(&adapter->hw, E1000_TDH(i)), + E1000_READ_REG(&adapter->hw, E1000_TDT(i))); + device_printf(dev, "Tx Queue Status = %d\n", txr->busy); + device_printf(dev, "TX descriptors avail = %d\n", + txr->tx_avail); + device_printf(dev, "Tx Descriptors avail failure = %ld\n", + txr->no_desc_avail); + device_printf(dev, "RX Queue %d ------\n", i); + device_printf(dev, "hw rdh = %d, hw rdt = %d\n", + E1000_READ_REG(&adapter->hw, E1000_RDH(i)), + E1000_READ_REG(&adapter->hw, E1000_RDT(i))); + device_printf(dev, "RX discarded packets = %ld\n", + rxr->rx_discarded); + device_printf(dev, "RX Next to Check = %d\n", rxr->next_to_check); + device_printf(dev, "RX Next to Refresh = %d\n", rxr->next_to_refresh); + } +} + +#ifdef EM_MULTIQUEUE +/* + * 82574 only: + * Write a new value to the EEPROM increasing the number of MSIX + * vectors from 3 to 5, for proper multiqueue support. + */ +static void +em_enable_vectors_82574(struct adapter *adapter) +{ + struct e1000_hw *hw = &adapter->hw; + device_t dev = adapter->dev; + u16 edata; + + e1000_read_nvm(hw, EM_NVM_PCIE_CTRL, 1, &edata); + printf("Current cap: %#06x\n", edata); + if (((edata & EM_NVM_MSIX_N_MASK) >> EM_NVM_MSIX_N_SHIFT) != 4) { + device_printf(dev, "Writing to eeprom: increasing " + "reported MSIX vectors from 3 to 5...\n"); + edata &= ~(EM_NVM_MSIX_N_MASK); + edata |= 4 << EM_NVM_MSIX_N_SHIFT; + e1000_write_nvm(hw, EM_NVM_PCIE_CTRL, 1, &edata); + e1000_update_nvm_checksum(hw); + device_printf(dev, "Writing to eeprom: done\n"); + } +} +#endif + +#ifdef DDB +DB_COMMAND(em_reset_dev, em_ddb_reset_dev) +{ + devclass_t dc; + int max_em; + + dc = devclass_find("em"); + max_em = devclass_get_maxunit(dc); + + for (int index = 0; index < (max_em - 1); index++) { + device_t dev; + dev = devclass_get_device(dc, index); + if (device_get_driver(dev) == &em_driver) { + struct adapter *adapter = device_get_softc(dev); + EM_CORE_LOCK(adapter); + em_init_locked(adapter); + EM_CORE_UNLOCK(adapter); + } + } } +DB_COMMAND(em_dump_queue, em_ddb_dump_queue) +{ + devclass_t dc; + int max_em; + + dc = devclass_find("em"); + max_em = devclass_get_maxunit(dc); + + for (int index = 0; index < (max_em - 1); index++) { + device_t dev; + dev = devclass_get_device(dc, index); + if (device_get_driver(dev) == &em_driver) + em_print_debug_info(device_get_softc(dev)); + } + +} +#endif diff --git a/freebsd/sys/dev/e1000/if_em.h b/freebsd/sys/dev/e1000/if_em.h index 09d81d25..2a2bf2cc 100644 --- a/freebsd/sys/dev/e1000/if_em.h +++ b/freebsd/sys/dev/e1000/if_em.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2011, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -53,7 +53,11 @@ */ #define EM_MIN_TXD 80 #define EM_MAX_TXD 4096 +#ifdef EM_MULTIQUEUE +#define EM_DEFAULT_TXD 4096 +#else #define EM_DEFAULT_TXD 1024 +#endif /* * EM_RXD - Maximum number of receive Descriptors @@ -70,7 +74,11 @@ */ #define EM_MIN_RXD 80 #define EM_MAX_RXD 4096 +#ifdef EM_MULTIQUEUE +#define EM_DEFAULT_RXD 4096 +#else #define EM_DEFAULT_RXD 1024 +#endif /* * EM_TIDV - Transmit Interrupt Delay Value @@ -117,7 +125,11 @@ * restoring the network connection. To eliminate the potential * for the hang ensure that EM_RDTR is set to 0. */ +#ifdef EM_MULTIQUEUE +#define EM_RDTR 64 +#else #define EM_RDTR 0 +#endif /* * Receive Interrupt Absolute Delay Timer (Not valid for 82542/82543/82544) @@ -130,7 +142,11 @@ * along with EM_RDTR, may improve traffic throughput in specific network * conditions. */ +#ifdef EM_MULTIQUEUE +#define EM_RADV 128 +#else #define EM_RADV 64 +#endif /* * This parameter controls the max duration of transmit watchdog. @@ -188,9 +204,22 @@ #define EM_EEPROM_APME 0x400; #define EM_82544_APME 0x0004; -#define EM_QUEUE_IDLE 0 -#define EM_QUEUE_WORKING 1 -#define EM_QUEUE_HUNG 2 +/* + * Driver state logic for the detection of a hung state + * in hardware. Set TX_HUNG whenever a TX packet is used + * (data is sent) and clear it when txeof() is invoked if + * any descriptors from the ring are cleaned/reclaimed. + * Increment internal counter if no descriptors are cleaned + * and compare to TX_MAXTRIES. When counter > TX_MAXTRIES, + * reset adapter. + */ +#define EM_TX_IDLE 0x00000000 +#define EM_TX_BUSY 0x00000001 +#define EM_TX_HUNG 0x80000000 +#define EM_TX_MAXTRIES 10 + +#define PCICFG_DESC_RING_STATUS 0xe4 +#define FLUSH_DESC_REQUIRED 0x100 /* * TDBA/RDBA should be aligned on 16 byte boundary. But TDLEN/RDLEN should be @@ -199,7 +228,15 @@ */ #define EM_DBA_ALIGN 128 -#define SPEED_MODE_BIT (1<<21) /* On PCI-E MACs only */ +/* + * See Intel 82574 Driver Programming Interface Manual, Section 10.2.6.9 + */ +#define TARC_COMPENSATION_MODE (1 << 7) /* Compensation Mode */ +#define TARC_SPEED_MODE_BIT (1 << 21) /* On PCI-E MACs only */ +#define TARC_MQ_FIX (1 << 23) | \ + (1 << 24) | \ + (1 << 25) /* Handle errata in MQ mode */ +#define TARC_ERRATA_BIT (1 << 26) /* Note from errata on 82574 */ /* PCI Config defines */ #define EM_BAR_TYPE(v) ((v) & EM_BAR_TYPE_MASK) @@ -232,7 +269,7 @@ #define HW_DEBUGOUT1(S, A) if (DEBUG_HW) printf(S "\n", A) #define HW_DEBUGOUT2(S, A, B) if (DEBUG_HW) printf(S "\n", A, B) -#define EM_MAX_SCATTER 32 +#define EM_MAX_SCATTER 40 #define EM_VFTA_SIZE 128 #define EM_TSO_SIZE (65535 + sizeof(struct ether_vlan_header)) #define EM_TSO_SEG_SIZE 4096 /* Max dma segment size */ @@ -249,6 +286,14 @@ * solve it just using this define. */ #define EM_EIAC 0x000DC +/* + * 82574 only reports 3 MSI-X vectors by default; + * defines assisting with making it report 5 are + * located here. + */ +#define EM_NVM_PCIE_CTRL 0x1B +#define EM_NVM_MSIX_N_MASK (0x7 << EM_NVM_MSIX_N_SHIFT) +#define EM_NVM_MSIX_N_SHIFT 7 /* * Bus dma allocation structure used by @@ -281,15 +326,14 @@ struct tx_ring { u32 me; u32 msix; u32 ims; - int queue_status; - int watchdog_time; + int busy; struct em_dma_alloc txdma; struct e1000_tx_desc *tx_base; struct task tx_task; struct taskqueue *tq; u32 next_avail_desc; u32 next_to_clean; - struct em_buffer *tx_buffers; + struct em_txbuffer *tx_buffers; volatile u16 tx_avail; u32 tx_tso; /* last tx was tso */ u16 last_hw_offload; @@ -321,11 +365,11 @@ struct rx_ring { u32 payload; struct task rx_task; struct taskqueue *tq; - struct e1000_rx_desc *rx_base; + union e1000_rx_desc_extended *rx_base; struct em_dma_alloc rxdma; u32 next_to_refresh; u32 next_to_check; - struct em_buffer *rx_buffers; + struct em_rxbuffer *rx_buffers; struct mbuf *fmp; struct mbuf *lmp; @@ -345,12 +389,12 @@ struct rx_ring { /* Our adapter structure */ struct adapter { - struct ifnet *ifp; + if_t ifp; struct e1000_hw hw; /* FreeBSD operating-system-specific structures. */ struct e1000_osdep osdep; - struct device *dev; + device_t dev; struct cdev *led_dev; struct resource *memory; @@ -368,7 +412,6 @@ struct adapter { int if_flags; int max_frame_size; int min_frame_size; - int pause_frames; struct mtx core_mtx; int em_insert_vlan_header; u32 ims; @@ -383,7 +426,7 @@ struct adapter { eventhandler_tag vlan_detach; u16 num_vlans; - u16 num_queues; + u8 num_queues; /* * Transmit rings: @@ -433,13 +476,12 @@ struct adapter { /* Misc stats maintained by the driver */ unsigned long dropped_pkts; - unsigned long mbuf_alloc_failed; - unsigned long mbuf_cluster_failed; + unsigned long link_irq; + unsigned long mbuf_defrag_failed; + unsigned long no_tx_dma_setup; unsigned long no_tx_map_avail; - unsigned long no_tx_dma_setup; unsigned long rx_overruns; unsigned long watchdog_events; - unsigned long link_irq; struct e1000_hw_stats stats; }; @@ -459,10 +501,17 @@ typedef struct _em_vendor_info_t { unsigned int index; } em_vendor_info_t; -struct em_buffer { +struct em_txbuffer { + int next_eop; /* Index of the desc to watch */ + struct mbuf *m_head; + bus_dmamap_t map; /* bus_dma map for packet */ +}; + +struct em_rxbuffer { int next_eop; /* Index of the desc to watch */ struct mbuf *m_head; bus_dmamap_t map; /* bus_dma map for packet */ + bus_addr_t paddr; }; @@ -501,4 +550,9 @@ e1000_rx_unrefreshed(struct rx_ring *rxr) #define EM_TX_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->tx_mtx, MA_OWNED) #define EM_RX_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->rx_mtx, MA_OWNED) +#define EM_RSSRK_SIZE 4 +#define EM_RSSRK_VAL(key, i) (key[(i) * EM_RSSRK_SIZE] | \ + key[(i) * EM_RSSRK_SIZE + 1] << 8 | \ + key[(i) * EM_RSSRK_SIZE + 2] << 16 | \ + key[(i) * EM_RSSRK_SIZE + 3] << 24) #endif /* _EM_H_DEFINED_ */ diff --git a/freebsd/sys/dev/e1000/if_igb.c b/freebsd/sys/dev/e1000/if_igb.c index 27ae386c..d683b85f 100644 --- a/freebsd/sys/dev/e1000/if_igb.c +++ b/freebsd/sys/dev/e1000/if_igb.c @@ -2,7 +2,7 @@ /****************************************************************************** - Copyright (c) 2001-2013, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -37,73 +37,19 @@ #include <rtems/bsd/local/opt_inet.h> #include <rtems/bsd/local/opt_inet6.h> +#include <rtems/bsd/local/opt_rss.h> #ifdef HAVE_KERNEL_OPTION_HEADERS #include <rtems/bsd/local/opt_device_polling.h> #include <rtems/bsd/local/opt_altq.h> #endif -#include <rtems/bsd/sys/param.h> -#include <sys/systm.h> -#ifndef IGB_LEGACY_TX -#include <sys/buf_ring.h> -#endif -#include <sys/bus.h> -#include <sys/endian.h> -#include <sys/kernel.h> -#include <sys/kthread.h> -#include <sys/malloc.h> -#include <sys/mbuf.h> -#include <sys/module.h> -#include <sys/rman.h> -#include <sys/socket.h> -#include <sys/sockio.h> -#include <sys/sysctl.h> -#include <sys/taskqueue.h> -#include <sys/eventhandler.h> -#include <sys/pcpu.h> -#include <sys/smp.h> -#include <machine/smp.h> -#include <machine/bus.h> -#include <machine/resource.h> - -#include <net/bpf.h> -#include <net/ethernet.h> -#include <net/if.h> -#include <net/if_arp.h> -#include <net/if_dl.h> -#include <net/if_media.h> - -#include <net/if_types.h> -#include <net/if_vlan_var.h> - -#include <netinet/in_systm.h> -#include <netinet/in.h> -#include <netinet/if_ether.h> -#include <netinet/ip.h> -#include <netinet/ip6.h> -#include <netinet/tcp.h> -#include <netinet/tcp_lro.h> -#include <netinet/udp.h> - -#include <machine/in_cksum.h> -#include <dev/led/led.h> -#include <dev/pci/pcivar.h> -#include <dev/pci/pcireg.h> - -#include "e1000_api.h" -#include "e1000_82575.h" #include "if_igb.h" /********************************************************************* - * Set this to one to display debug statistics - *********************************************************************/ -int igb_display_debug_stats = 0; - -/********************************************************************* * Driver version: *********************************************************************/ -char igb_driver_version[] = "version - 2.4.0"; +char igb_driver_version[] = "2.5.3-k"; /********************************************************************* @@ -118,60 +64,47 @@ char igb_driver_version[] = "version - 2.4.0"; static igb_vendor_info_t igb_vendor_info_array[] = { - { 0x8086, E1000_DEV_ID_82575EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82576, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82576_NS, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82576_NS_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82576_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82576_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82576_SERDES_QUAD, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82576_QUAD_COPPER, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82576_QUAD_COPPER_ET2, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82576_VF, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82580_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82580_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82580_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82580_SGMII, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82580_COPPER_DUAL, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82580_QUAD_FIBER, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_DH89XXCC_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_DH89XXCC_SGMII, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_DH89XXCC_SFP, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_DH89XXCC_BACKPLANE, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_I350_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_I350_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_I350_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_I350_SGMII, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_I350_VF, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_I210_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_I210_COPPER_IT, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_I210_COPPER_OEM1, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_I210_COPPER_FLASHLESS, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_I210_SERDES_FLASHLESS, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_I210_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_I210_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_I210_SGMII, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_I211_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_I354_BACKPLANE_1GBPS, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_I354_SGMII, PCI_ANY_ID, PCI_ANY_ID, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_COPPER, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_FIBER_SERDES, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575GB_QUAD_COPPER, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS_SERDES, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_FIBER, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES_QUAD, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER_ET2, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_FIBER, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SERDES, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SGMII, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER_DUAL, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_QUAD_FIBER, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SERDES, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SGMII, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SFP, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_BACKPLANE, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_COPPER, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_FIBER, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SERDES, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SGMII, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_IT, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_OEM1, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_FLASHLESS, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES_FLASHLESS, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_FIBER, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SGMII, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I211_COPPER, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_1GBPS, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS, 0, 0, 0}, + {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_SGMII, 0, 0, 0}, /* required last entry */ - { 0, 0, 0, 0, 0} + {0, 0, 0, 0, 0} }; /********************************************************************* @@ -201,6 +134,7 @@ static void igb_start(struct ifnet *); static void igb_start_locked(struct tx_ring *, struct ifnet *ifp); #endif static int igb_ioctl(struct ifnet *, u_long, caddr_t); +static uint64_t igb_get_counter(if_t, ift_counter); static void igb_init(void *); static void igb_init_locked(struct adapter *); static void igb_stop(void *); @@ -318,6 +252,9 @@ static devclass_t igb_devclass; DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0); MODULE_DEPEND(igb, pci, 1, 1, 1); MODULE_DEPEND(igb, ether, 1, 1, 1); +#ifdef DEV_NETMAP +MODULE_DEPEND(igb, netmap, 1, 1, 1); +#endif /* DEV_NETMAP */ /********************************************************************* * Tunable default values. @@ -328,8 +265,6 @@ static SYSCTL_NODE(_hw, OID_AUTO, igb, CTLFLAG_RD, 0, "IGB driver parameters"); /* Descriptor defaults */ static int igb_rxd = IGB_DEFAULT_RXD; static int igb_txd = IGB_DEFAULT_TXD; -TUNABLE_INT("hw.igb.rxd", &igb_rxd); -TUNABLE_INT("hw.igb.txd", &igb_txd); SYSCTL_INT(_hw_igb, OID_AUTO, rxd, CTLFLAG_RDTUN, &igb_rxd, 0, "Number of receive descriptors per queue"); SYSCTL_INT(_hw_igb, OID_AUTO, txd, CTLFLAG_RDTUN, &igb_txd, 0, @@ -342,8 +277,7 @@ SYSCTL_INT(_hw_igb, OID_AUTO, txd, CTLFLAG_RDTUN, &igb_txd, 0, ** traffic for that interrupt vector */ static int igb_enable_aim = TRUE; -TUNABLE_INT("hw.igb.enable_aim", &igb_enable_aim); -SYSCTL_INT(_hw_igb, OID_AUTO, enable_aim, CTLFLAG_RW, &igb_enable_aim, 0, +SYSCTL_INT(_hw_igb, OID_AUTO, enable_aim, CTLFLAG_RWTUN, &igb_enable_aim, 0, "Enable adaptive interrupt moderation"); /* @@ -351,7 +285,6 @@ SYSCTL_INT(_hw_igb, OID_AUTO, enable_aim, CTLFLAG_RW, &igb_enable_aim, 0, * but this allows it to be forced off for testing. */ static int igb_enable_msix = 1; -TUNABLE_INT("hw.igb.enable_msix", &igb_enable_msix); SYSCTL_INT(_hw_igb, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &igb_enable_msix, 0, "Enable MSI-X interrupts"); @@ -359,7 +292,6 @@ SYSCTL_INT(_hw_igb, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &igb_enable_msix, 0, ** Tuneable Interrupt rate */ static int igb_max_interrupt_rate = 8000; -TUNABLE_INT("hw.igb.max_interrupt_rate", &igb_max_interrupt_rate); SYSCTL_INT(_hw_igb, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN, &igb_max_interrupt_rate, 0, "Maximum interrupts per second"); @@ -368,21 +300,19 @@ SYSCTL_INT(_hw_igb, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN, ** Tuneable number of buffers in the buf-ring (drbr_xxx) */ static int igb_buf_ring_size = IGB_BR_SIZE; -TUNABLE_INT("hw.igb.buf_ring_size", &igb_buf_ring_size); SYSCTL_INT(_hw_igb, OID_AUTO, buf_ring_size, CTLFLAG_RDTUN, &igb_buf_ring_size, 0, "Size of the bufring"); #endif /* ** Header split causes the packet header to -** be dma'd to a seperate mbuf from the payload. +** be dma'd to a separate mbuf from the payload. ** this can have memory alignment benefits. But ** another plus is that small packets often fit ** into the header and thus use no cluster. Its ** a very workload dependent type feature. */ static int igb_header_split = FALSE; -TUNABLE_INT("hw.igb.hdr_split", &igb_header_split); SYSCTL_INT(_hw_igb, OID_AUTO, header_split, CTLFLAG_RDTUN, &igb_header_split, 0, "Enable receive mbuf header split"); @@ -392,7 +322,6 @@ SYSCTL_INT(_hw_igb, OID_AUTO, header_split, CTLFLAG_RDTUN, &igb_header_split, 0, ** MSIX messages if left at 0. */ static int igb_num_queues = 0; -TUNABLE_INT("hw.igb.num_queues", &igb_num_queues); SYSCTL_INT(_hw_igb, OID_AUTO, num_queues, CTLFLAG_RDTUN, &igb_num_queues, 0, "Number of queues to configure, 0 indicates autoconfigure"); @@ -405,11 +334,16 @@ static int igb_last_bind_cpu = -1; /* How many packets rxeof tries to clean at a time */ static int igb_rx_process_limit = 100; -TUNABLE_INT("hw.igb.rx_process_limit", &igb_rx_process_limit); SYSCTL_INT(_hw_igb, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, &igb_rx_process_limit, 0, "Maximum number of received packets to process at a time, -1 means unlimited"); +/* How many packets txeof tries to clean at a time */ +static int igb_tx_process_limit = -1; +SYSCTL_INT(_hw_igb, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN, + &igb_tx_process_limit, 0, + "Maximum number of sent packets to process at a time, -1 means unlimited"); + #ifdef DEV_NETMAP /* see ixgbe.c for details */ #include <dev/netmap/if_igb_netmap.h> #endif /* DEV_NETMAP */ @@ -425,7 +359,7 @@ SYSCTL_INT(_hw_igb, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, static int igb_probe(device_t dev) { - char adapter_name[60]; + char adapter_name[256]; uint16_t pci_vendor_id = 0; uint16_t pci_device_id = 0; uint16_t pci_subvendor_id = 0; @@ -435,7 +369,7 @@ igb_probe(device_t dev) INIT_DEBUGOUT("igb_probe: begin"); pci_vendor_id = pci_get_vendor(dev); - if (pci_vendor_id != IGB_VENDOR_ID) + if (pci_vendor_id != IGB_INTEL_VENDOR_ID) return (ENXIO); pci_device_id = pci_get_device(dev); @@ -448,11 +382,11 @@ igb_probe(device_t dev) (pci_device_id == ent->device_id) && ((pci_subvendor_id == ent->subvendor_id) || - (ent->subvendor_id == PCI_ANY_ID)) && + (ent->subvendor_id == 0)) && ((pci_subdevice_id == ent->subdevice_id) || - (ent->subdevice_id == PCI_ANY_ID))) { - sprintf(adapter_name, "%s %s", + (ent->subdevice_id == 0))) { + sprintf(adapter_name, "%s, Version - %s", igb_strings[ent->index], igb_driver_version); device_set_desc_copy(dev, adapter_name); @@ -460,7 +394,6 @@ igb_probe(device_t dev) } ent++; } - return (ENXIO); } @@ -492,7 +425,7 @@ igb_attach(device_t dev) adapter->dev = adapter->osdep.dev = dev; IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); - /* SYSCTL stuff */ + /* SYSCTLs */ SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, @@ -528,11 +461,15 @@ igb_attach(device_t dev) e1000_get_bus_info(&adapter->hw); - /* Sysctl for limiting the amount of work done in the taskqueue */ + /* Sysctls for limiting the amount of work done in the taskqueues */ igb_set_sysctl_value(adapter, "rx_processing_limit", "max number of rx packets to process", &adapter->rx_process_limit, igb_rx_process_limit); + igb_set_sysctl_value(adapter, "tx_processing_limit", + "max number of tx packets to process", + &adapter->tx_process_limit, igb_tx_process_limit); + /* * Validate number of transmit and receive descriptors. It * must not exceed hardware maximum, and must be multiple @@ -616,9 +553,9 @@ igb_attach(device_t dev) "Disable Energy Efficient Ethernet"); if (adapter->hw.phy.media_type == e1000_media_type_copper) { if (adapter->hw.mac.type == e1000_i354) - e1000_set_eee_i354(&adapter->hw); + e1000_set_eee_i354(&adapter->hw, TRUE, TRUE); else - e1000_set_eee_i350(&adapter->hw); + e1000_set_eee_i350(&adapter->hw, TRUE, TRUE); } } @@ -724,7 +661,8 @@ igb_attach(device_t dev) return (0); err_late: - igb_detach(dev); + if (igb_detach(dev) == 0) /* igb_detach() already did the cleanup */ + return(error); igb_free_transmit_structures(adapter); igb_free_receive_structures(adapter); igb_release_hw_control(adapter); @@ -977,12 +915,33 @@ igb_mq_start(struct ifnet *ifp, struct mbuf *m) struct igb_queue *que; struct tx_ring *txr; int i, err = 0; +#ifdef RSS + uint32_t bucket_id; +#endif /* Which queue to use */ - if ((m->m_flags & M_FLOWID) != 0) - i = m->m_pkthdr.flowid % adapter->num_queues; - else + /* + * When doing RSS, map it to the same outbound queue + * as the incoming flow would be mapped to. + * + * If everything is setup correctly, it should be the + * same bucket that the current CPU we're on is. + */ + if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { +#ifdef RSS + if (rss_hash2bucket(m->m_pkthdr.flowid, + M_HASHTYPE_GET(m), &bucket_id) == 0) { + /* XXX TODO: spit out something if bucket_id > num_queues? */ + i = bucket_id % adapter->num_queues; + } else { +#endif + i = m->m_pkthdr.flowid % adapter->num_queues; +#ifdef RSS + } +#endif + } else { i = curcpu % adapter->num_queues; + } txr = &adapter->tx_rings[i]; que = &adapter->queues[i]; @@ -1011,7 +970,6 @@ igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr) adapter->link_active == 0) return (ENETDOWN); - /* Process the queue */ while ((next = drbr_peek(ifp, txr->br)) != NULL) { if ((err = igb_xmit(txr, &next)) != 0) { @@ -1030,9 +988,8 @@ igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr) } drbr_advance(ifp, txr->br); enq++; - ifp->if_obytes += next->m_pkthdr.len; - if (next->m_flags & M_MCAST) - ifp->if_omcasts++; + if (next->m_flags & M_MCAST && adapter->vf_ifp) + if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); ETHER_BPF_MTAP(ifp, next); if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) break; @@ -1151,7 +1108,8 @@ igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data) ifp->if_mtu = ifr->ifr_mtu; adapter->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; - igb_init_locked(adapter); + if (ifp->if_drv_flags & IFF_DRV_RUNNING) + igb_init_locked(adapter); IGB_CORE_UNLOCK(adapter); break; } @@ -1230,10 +1188,27 @@ igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data) } } #endif +#if __FreeBSD_version >= 1000000 + /* HW cannot turn these on/off separately */ + if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) { + ifp->if_capenable ^= IFCAP_RXCSUM; + ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; + reinit = 1; + } + if (mask & IFCAP_TXCSUM) { + ifp->if_capenable ^= IFCAP_TXCSUM; + reinit = 1; + } + if (mask & IFCAP_TXCSUM_IPV6) { + ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; + reinit = 1; + } +#else if (mask & IFCAP_HWCSUM) { ifp->if_capenable ^= IFCAP_HWCSUM; reinit = 1; } +#endif if (mask & IFCAP_TSO4) { ifp->if_capenable ^= IFCAP_TSO4; reinit = 1; @@ -1312,16 +1287,32 @@ igb_init_locked(struct adapter *adapter) /* Set hardware offload abilities */ ifp->if_hwassist = 0; if (ifp->if_capenable & IFCAP_TXCSUM) { +#if __FreeBSD_version >= 1000000 + ifp->if_hwassist |= (CSUM_IP_TCP | CSUM_IP_UDP); + if (adapter->hw.mac.type != e1000_82575) + ifp->if_hwassist |= CSUM_IP_SCTP; +#else ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); #if __FreeBSD_version >= 800000 - if (adapter->hw.mac.type == e1000_82576) + if (adapter->hw.mac.type != e1000_82575) ifp->if_hwassist |= CSUM_SCTP; #endif +#endif } +#if __FreeBSD_version >= 1000000 + if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) { + ifp->if_hwassist |= (CSUM_IP6_TCP | CSUM_IP6_UDP); + if (adapter->hw.mac.type != e1000_82575) + ifp->if_hwassist |= CSUM_IP6_SCTP; + } +#endif if (ifp->if_capenable & IFCAP_TSO) ifp->if_hwassist |= CSUM_TSO; + /* Clear bad data from Rx FIFOs */ + e1000_rx_fifo_flush_82575(&adapter->hw); + /* Configure for OS presence */ igb_init_manageability(adapter); @@ -1385,9 +1376,9 @@ igb_init_locked(struct adapter *adapter) /* Set Energy Efficient Ethernet */ if (adapter->hw.phy.media_type == e1000_media_type_copper) { if (adapter->hw.mac.type == e1000_i354) - e1000_set_eee_i354(&adapter->hw); + e1000_set_eee_i354(&adapter->hw, TRUE, TRUE); else - e1000_set_eee_i350(&adapter->hw); + e1000_set_eee_i350(&adapter->hw, TRUE, TRUE); } } @@ -1879,7 +1870,8 @@ retry: /* Try it again? - one try */ if (remap == TRUE) { remap = FALSE; - m = m_defrag(*m_headp, M_NOWAIT); + m = m_collapse(*m_headp, M_NOWAIT, + IGB_MAX_SCATTER); if (m == NULL) { adapter->mbuf_defrag_failed++; m_freem(*m_headp); @@ -1890,9 +1882,6 @@ retry: goto retry; } else return (error); - case ENOMEM: - txr->no_tx_dma_setup++; - return (error); default: txr->no_tx_dma_setup++; m_freem(*m_headp); @@ -1902,7 +1891,7 @@ retry: } /* Make certain there are enough descriptors */ - if (nsegs > txr->tx_avail - 2) { + if (txr->tx_avail < (nsegs + 2)) { txr->no_desc_avail++; bus_dmamap_unload(txr->txtag, map); return (ENOBUFS); @@ -2443,11 +2432,37 @@ igb_allocate_msix(struct adapter *adapter) device_t dev = adapter->dev; struct igb_queue *que = adapter->queues; int error, rid, vector = 0; + int cpu_id = 0; +#ifdef RSS + cpuset_t cpu_mask; +#endif /* Be sure to start with all interrupts disabled */ E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0); E1000_WRITE_FLUSH(&adapter->hw); +#ifdef RSS + /* + * If we're doing RSS, the number of queues needs to + * match the number of RSS buckets that are configured. + * + * + If there's more queues than RSS buckets, we'll end + * up with queues that get no traffic. + * + * + If there's more RSS buckets than queues, we'll end + * up having multiple RSS buckets map to the same queue, + * so there'll be some contention. + */ + if (adapter->num_queues != rss_getnumbuckets()) { + device_printf(dev, + "%s: number of queues (%d) != number of RSS buckets (%d)" + "; performance will be impacted.\n", + __func__, + adapter->num_queues, + rss_getnumbuckets()); + } +#endif + for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { rid = vector +1; que->res = bus_alloc_resource_any(dev, @@ -2474,19 +2489,42 @@ igb_allocate_msix(struct adapter *adapter) que->eims = E1000_EICR_TX_QUEUE0 << i; else que->eims = 1 << vector; + +#ifdef RSS /* - ** Bind the msix vector, and thus the - ** rings to the corresponding cpu. - */ + * The queue ID is used as the RSS layer bucket ID. + * We look up the queue ID -> RSS CPU ID and select + * that. + */ + cpu_id = rss_getcpu(i % rss_getnumbuckets()); +#else + /* + * Bind the msix vector, and thus the + * rings to the corresponding cpu. + * + * This just happens to match the default RSS round-robin + * bucket -> queue -> CPU allocation. + */ if (adapter->num_queues > 1) { if (igb_last_bind_cpu < 0) igb_last_bind_cpu = CPU_FIRST(); - bus_bind_intr(dev, que->res, igb_last_bind_cpu); + cpu_id = igb_last_bind_cpu; + } +#endif + + if (adapter->num_queues > 1) { + bus_bind_intr(dev, que->res, cpu_id); +#ifdef RSS + device_printf(dev, + "Bound queue %d to RSS bucket %d\n", + i, cpu_id); +#else device_printf(dev, "Bound queue %d to cpu %d\n", - i,igb_last_bind_cpu); - igb_last_bind_cpu = CPU_NEXT(igb_last_bind_cpu); + i, cpu_id); +#endif } + #ifndef IGB_LEGACY_TX TASK_INIT(&que->txr->txq_task, 0, igb_deferred_mq_start, que->txr); @@ -2495,8 +2533,35 @@ igb_allocate_msix(struct adapter *adapter) TASK_INIT(&que->que_task, 0, igb_handle_que, que); que->tq = taskqueue_create("igb_que", M_NOWAIT, taskqueue_thread_enqueue, &que->tq); - taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que", - device_get_nameunit(adapter->dev)); + if (adapter->num_queues > 1) { + /* + * Only pin the taskqueue thread to a CPU if + * RSS is in use. + * + * This again just happens to match the default RSS + * round-robin bucket -> queue -> CPU allocation. + */ +#ifdef RSS + CPU_SETOF(cpu_id, &cpu_mask); + taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET, + &cpu_mask, + "%s que (bucket %d)", + device_get_nameunit(adapter->dev), + cpu_id); +#else + taskqueue_start_threads(&que->tq, 1, PI_NET, + "%s que (qid %d)", + device_get_nameunit(adapter->dev), + cpu_id); +#endif + } else { + taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que", + device_get_nameunit(adapter->dev)); + } + + /* Finally update the last bound CPU id */ + if (adapter->num_queues > 1) + igb_last_bind_cpu = CPU_NEXT(igb_last_bind_cpu); } /* And Link */ @@ -2770,13 +2835,19 @@ igb_setup_msix(struct adapter *adapter) goto msi; } - /* Figure out a reasonable auto config value */ queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus; - /* Manual override */ + /* Override via tuneable */ if (igb_num_queues != 0) queues = igb_num_queues; +#ifdef RSS + /* If we're doing RSS, clamp at the number of RSS buckets */ + if (queues > rss_getnumbuckets()) + queues = rss_getnumbuckets(); +#endif + + /* Sanity check based on HW */ switch (adapter->hw.mac.type) { case e1000_82575: @@ -2798,13 +2869,11 @@ igb_setup_msix(struct adapter *adapter) maxqueues = 1; break; } + + /* Final clamp on the actual hardware capability */ if (queues > maxqueues) queues = maxqueues; - /* Manual override */ - if (igb_num_queues != 0) - queues = igb_num_queues; - /* ** One vector (RX/TX pair) per queue ** plus an additional for Link interrupt @@ -2913,21 +2982,6 @@ igb_init_dmac(struct adapter *adapter, u32 pba) E1000_WRITE_REG(hw, E1000_DMACR, reg); -#ifdef I210_OBFF_SUPPORT - /* - * Set the OBFF Rx threshold to DMA Coalescing Rx - * threshold - 2KB and enable the feature in the - * hardware for I210. - */ - if (hw->mac.type == e1000_i210) { - int obff = dmac - 2; - reg = E1000_READ_REG(hw, E1000_DOBFFCTL); - reg &= ~E1000_DOBFFCTL_OBFFTHR_MASK; - reg |= (obff & E1000_DOBFFCTL_OBFFTHR_MASK) - | E1000_DOBFFCTL_EXIT_ACT_MASK; - E1000_WRITE_REG(hw, E1000_DOBFFCTL, reg); - } -#endif E1000_WRITE_REG(hw, E1000_DMCRTRH, 0); /* Set the interval before transition */ @@ -3117,6 +3171,13 @@ igb_setup_interface(device_t dev, struct adapter *adapter) ifp->if_softc = adapter; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_ioctl = igb_ioctl; + ifp->if_get_counter = igb_get_counter; + + /* TSO parameters */ + ifp->if_hw_tsomax = IP_MAXPACKET; + ifp->if_hw_tsomaxsegcount = IGB_MAX_SCATTER; + ifp->if_hw_tsomaxsegsize = IGB_TSO_SEG_SIZE; + #ifndef IGB_LEGACY_TX ifp->if_transmit = igb_mq_start; ifp->if_qflush = igb_qflush; @@ -3132,6 +3193,9 @@ igb_setup_interface(device_t dev, struct adapter *adapter) ifp->if_capabilities = ifp->if_capenable = 0; ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; +#if __FreeBSD_version >= 1000000 + ifp->if_capabilities |= IFCAP_HWCSUM_IPV6; +#endif ifp->if_capabilities |= IFCAP_TSO; ifp->if_capabilities |= IFCAP_JUMBO_MTU; ifp->if_capenable = ifp->if_capabilities; @@ -3147,7 +3211,7 @@ igb_setup_interface(device_t dev, struct adapter *adapter) * Tell the upper layer(s) we * support full VLAN capability. */ - ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); + ifp->if_hdrlen = sizeof(struct ether_vlan_header); ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWTSO | IFCAP_VLAN_MTU; @@ -3260,7 +3324,6 @@ fail_2: bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); bus_dma_tag_destroy(dma->dma_tag); fail_0: - dma->dma_map = NULL; dma->dma_tag = NULL; return (error); @@ -3271,12 +3334,15 @@ igb_dma_free(struct adapter *adapter, struct igb_dma_alloc *dma) { if (dma->dma_tag == NULL) return; - if (dma->dma_map != NULL) { + if (dma->dma_paddr != 0) { bus_dmamap_sync(dma->dma_tag, dma->dma_map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(dma->dma_tag, dma->dma_map); + dma->dma_paddr = 0; + } + if (dma->dma_vaddr != NULL) { bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); - dma->dma_map = NULL; + dma->dma_vaddr = NULL; } bus_dma_tag_destroy(dma->dma_tag); dma->dma_tag = NULL; @@ -3533,7 +3599,7 @@ igb_setup_transmit_ring(struct tx_ring *txr) if (slot) { int si = netmap_idx_n2k(&na->tx_rings[txr->me], i); /* no need to set the address */ - netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si)); + netmap_load_map(na, txr->txtag, txbuf->map, NMB(na, slot + si)); } #endif /* DEV_NETMAP */ /* clear the watch index */ @@ -3903,17 +3969,29 @@ igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, switch (ipproto) { case IPPROTO_TCP: +#if __FreeBSD_version >= 1000000 + if (mp->m_pkthdr.csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP)) +#else if (mp->m_pkthdr.csum_flags & CSUM_TCP) +#endif type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; break; case IPPROTO_UDP: +#if __FreeBSD_version >= 1000000 + if (mp->m_pkthdr.csum_flags & (CSUM_IP_UDP | CSUM_IP6_UDP)) +#else if (mp->m_pkthdr.csum_flags & CSUM_UDP) +#endif type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP; break; #if __FreeBSD_version >= 800000 case IPPROTO_SCTP: +#if __FreeBSD_version >= 1000000 + if (mp->m_pkthdr.csum_flags & (CSUM_IP_SCTP | CSUM_IP6_SCTP)) +#else if (mp->m_pkthdr.csum_flags & CSUM_SCTP) +#endif type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP; break; #endif @@ -3956,9 +4034,11 @@ static bool igb_txeof(struct tx_ring *txr) { struct adapter *adapter = txr->adapter; +#ifdef DEV_NETMAP struct ifnet *ifp = adapter->ifp; +#endif /* DEV_NETMAP */ u32 work, processed = 0; - u16 limit = txr->process_limit; + int limit = adapter->tx_process_limit; struct igb_tx_buf *buf; union e1000_adv_tx_desc *txd; @@ -4031,7 +4111,6 @@ igb_txeof(struct tx_ring *txr) } ++txr->packets; ++processed; - ++ifp->if_opackets; txr->watchdog_time = ticks; /* Try the next packet */ @@ -4332,13 +4411,13 @@ igb_setup_receive_ring(struct rx_ring *rxr) rxbuf = &rxr->rx_buffers[j]; #ifdef DEV_NETMAP if (slot) { - /* slot sj is mapped to the i-th NIC-ring entry */ + /* slot sj is mapped to the j-th NIC-ring entry */ int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j); uint64_t paddr; void *addr; - addr = PNMB(slot + sj, &paddr); - netmap_load_map(rxr->ptag, rxbuf->pmap, addr); + addr = PNMB(na, slot + sj, &paddr); + netmap_load_map(na, rxr->ptag, rxbuf->pmap, addr); /* Update descriptor */ rxr->rx_base[j].read.pkt_addr = htole64(paddr); continue; @@ -4399,7 +4478,6 @@ skip_head: rxr->fmp = NULL; rxr->lmp = NULL; - rxr->discard = FALSE; bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); @@ -4408,7 +4486,7 @@ skip_head: ** Now set up the LRO interface, we ** also only do head split when LRO ** is enabled, since so often they - ** are undesireable in similar setups. + ** are undesirable in similar setups. */ if (ifp->if_capenable & IFCAP_LRO) { error = tcp_lro_init(lro); @@ -4463,6 +4541,106 @@ fail: return (ENOBUFS); } +/* + * Initialise the RSS mapping for NICs that support multiple transmit/ + * receive rings. + */ +static void +igb_initialise_rss_mapping(struct adapter *adapter) +{ + struct e1000_hw *hw = &adapter->hw; + int i; + int queue_id; + u32 reta; + u32 rss_key[10], mrqc, shift = 0; + + /* XXX? */ + if (adapter->hw.mac.type == e1000_82575) + shift = 6; + + /* + * The redirection table controls which destination + * queue each bucket redirects traffic to. + * Each DWORD represents four queues, with the LSB + * being the first queue in the DWORD. + * + * This just allocates buckets to queues using round-robin + * allocation. + * + * NOTE: It Just Happens to line up with the default + * RSS allocation method. + */ + + /* Warning FM follows */ + reta = 0; + for (i = 0; i < 128; i++) { +#ifdef RSS + queue_id = rss_get_indirection_to_bucket(i); + /* + * If we have more queues than buckets, we'll + * end up mapping buckets to a subset of the + * queues. + * + * If we have more buckets than queues, we'll + * end up instead assigning multiple buckets + * to queues. + * + * Both are suboptimal, but we need to handle + * the case so we don't go out of bounds + * indexing arrays and such. + */ + queue_id = queue_id % adapter->num_queues; +#else + queue_id = (i % adapter->num_queues); +#endif + /* Adjust if required */ + queue_id = queue_id << shift; + + /* + * The low 8 bits are for hash value (n+0); + * The next 8 bits are for hash value (n+1), etc. + */ + reta = reta >> 8; + reta = reta | ( ((uint32_t) queue_id) << 24); + if ((i & 3) == 3) { + E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta); + reta = 0; + } + } + + /* Now fill in hash table */ + + /* + * MRQC: Multiple Receive Queues Command + * Set queuing to RSS control, number depends on the device. + */ + mrqc = E1000_MRQC_ENABLE_RSS_8Q; + +#ifdef RSS + /* XXX ew typecasting */ + rss_getkey((uint8_t *) &rss_key); +#else + arc4rand(&rss_key, sizeof(rss_key), 0); +#endif + for (i = 0; i < 10; i++) + E1000_WRITE_REG_ARRAY(hw, + E1000_RSSRK(0), i, rss_key[i]); + + /* + * Configure the RSS fields to hash upon. + */ + mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | + E1000_MRQC_RSS_FIELD_IPV4_TCP); + mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 | + E1000_MRQC_RSS_FIELD_IPV6_TCP); + mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP | + E1000_MRQC_RSS_FIELD_IPV6_UDP); + mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | + E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); + + E1000_WRITE_REG(hw, E1000_MRQC, mrqc); +} + /********************************************************************* * * Enable receive unit. @@ -4519,6 +4697,18 @@ igb_initialize_receive_units(struct adapter *adapter) rctl |= E1000_RCTL_SZ_2048; } + /* + * If TX flow control is disabled and there's >1 queue defined, + * enable DROP. + * + * This drops frames rather than hanging the RX MAC for all queues. + */ + if ((adapter->num_queues > 1) && + (adapter->fc == e1000_fc_none || + adapter->fc == e1000_fc_rx_pause)) { + srrctl |= E1000_SRRCTL_DROP_EN; + } + /* Setup the Base and Length of the Rx Descriptor Rings */ for (int i = 0; i < adapter->num_queues; i++, rxr++) { u64 bus_addr = rxr->rxdma.dma_paddr; @@ -4546,39 +4736,9 @@ igb_initialize_receive_units(struct adapter *adapter) */ rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); if (adapter->num_queues >1) { - u32 random[10], mrqc, shift = 0; - union igb_reta { - u32 dword; - u8 bytes[4]; - } reta; - arc4rand(&random, sizeof(random), 0); - if (adapter->hw.mac.type == e1000_82575) - shift = 6; - /* Warning FM follows */ - for (int i = 0; i < 128; i++) { - reta.bytes[i & 3] = - (i % adapter->num_queues) << shift; - if ((i & 3) == 3) - E1000_WRITE_REG(hw, - E1000_RETA(i >> 2), reta.dword); - } - /* Now fill in hash table */ - mrqc = E1000_MRQC_ENABLE_RSS_4Q; - for (int i = 0; i < 10; i++) - E1000_WRITE_REG_ARRAY(hw, - E1000_RSSRK(0), i, random[i]); - - mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | - E1000_MRQC_RSS_FIELD_IPV4_TCP); - mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 | - E1000_MRQC_RSS_FIELD_IPV6_TCP); - mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP | - E1000_MRQC_RSS_FIELD_IPV6_UDP); - mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | - E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); - - E1000_WRITE_REG(hw, E1000_MRQC, mrqc); + /* rss setup */ + igb_initialise_rss_mapping(adapter); /* ** NOTE: Receive Full-Packet Checksum Offload @@ -4589,8 +4749,8 @@ igb_initialize_receive_units(struct adapter *adapter) rxcsum |= E1000_RXCSUM_PCSD; #if __FreeBSD_version >= 800000 /* For SCTP Offload */ - if ((hw->mac.type == e1000_82576) - && (ifp->if_capenable & IFCAP_RXCSUM)) + if ((hw->mac.type != e1000_82575) && + (ifp->if_capenable & IFCAP_RXCSUM)) rxcsum |= E1000_RXCSUM_CRCOFL; #endif } else { @@ -4598,7 +4758,7 @@ igb_initialize_receive_units(struct adapter *adapter) if (ifp->if_capenable & IFCAP_RXCSUM) { rxcsum |= E1000_RXCSUM_IPPCSE; #if __FreeBSD_version >= 800000 - if (adapter->hw.mac.type == e1000_82576) + if (adapter->hw.mac.type != e1000_82575) rxcsum |= E1000_RXCSUM_CRCOFL; #endif } else @@ -4818,7 +4978,6 @@ igb_rxeof(struct igb_queue *que, int count, int *done) struct rx_ring *rxr = que->rxr; struct ifnet *ifp = adapter->ifp; struct lro_ctrl *lro = &rxr->lro; - struct lro_entry *queued; int i, processed = 0, rxdone = 0; u32 ptype, staterr = 0; union e1000_adv_rx_desc *cur; @@ -4839,7 +4998,7 @@ igb_rxeof(struct igb_queue *que, int count, int *done) for (i = rxr->next_to_check; count != 0;) { struct mbuf *sendmp, *mh, *mp; struct igb_rx_buf *rxbuf; - u16 hlen, plen, hdr, vtag; + u16 hlen, plen, hdr, vtag, pkt_info; bool eop = FALSE; cur = &rxr->rx_base[i]; @@ -4861,17 +5020,19 @@ igb_rxeof(struct igb_queue *que, int count, int *done) else vtag = le16toh(cur->wb.upper.vlan); hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); + pkt_info = le16toh(cur->wb.lower.lo_dword.hs_rss.pkt_info); eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP); - /* Make sure all segments of a bad packet are discarded */ - if (((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0) || - (rxr->discard)) { + /* + * Free the frame (all segments) if we're at EOP and + * it's an error. + * + * The datasheet states that EOP + status is only valid for + * the final segment in a multi-segment frame. + */ + if (eop && ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0)) { adapter->dropped_pkts++; ++rxr->rx_discarded; - if (!eop) /* Catch subsequent segs */ - rxr->discard = TRUE; - else - rxr->discard = FALSE; igb_rx_discard(rxr, i); goto next_desc; } @@ -4945,7 +5106,6 @@ igb_rxeof(struct igb_queue *que, int count, int *done) if (eop) { rxr->fmp->m_pkthdr.rcvif = ifp; - ifp->if_ipackets++; rxr->rx_packets++; /* capture data for AIM */ rxr->packets++; @@ -4960,10 +5120,51 @@ igb_rxeof(struct igb_queue *que, int count, int *done) rxr->fmp->m_pkthdr.ether_vtag = vtag; rxr->fmp->m_flags |= M_VLANTAG; } + + /* + * In case of multiqueue, we have RXCSUM.PCSD bit set + * and never cleared. This means we have RSS hash + * available to be used. + */ + if (adapter->num_queues > 1) { + rxr->fmp->m_pkthdr.flowid = + le32toh(cur->wb.lower.hi_dword.rss); + switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) { + case E1000_RXDADV_RSSTYPE_IPV4_TCP: + M_HASHTYPE_SET(rxr->fmp, + M_HASHTYPE_RSS_TCP_IPV4); + break; + case E1000_RXDADV_RSSTYPE_IPV4: + M_HASHTYPE_SET(rxr->fmp, + M_HASHTYPE_RSS_IPV4); + break; + case E1000_RXDADV_RSSTYPE_IPV6_TCP: + M_HASHTYPE_SET(rxr->fmp, + M_HASHTYPE_RSS_TCP_IPV6); + break; + case E1000_RXDADV_RSSTYPE_IPV6_EX: + M_HASHTYPE_SET(rxr->fmp, + M_HASHTYPE_RSS_IPV6_EX); + break; + case E1000_RXDADV_RSSTYPE_IPV6: + M_HASHTYPE_SET(rxr->fmp, + M_HASHTYPE_RSS_IPV6); + break; + case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX: + M_HASHTYPE_SET(rxr->fmp, + M_HASHTYPE_RSS_TCP_IPV6_EX); + break; + default: + /* XXX fallthrough */ + M_HASHTYPE_SET(rxr->fmp, + M_HASHTYPE_OPAQUE_HASH); + } + } else { #ifndef IGB_LEGACY_TX - rxr->fmp->m_pkthdr.flowid = que->msix; - rxr->fmp->m_flags |= M_FLOWID; + rxr->fmp->m_pkthdr.flowid = que->msix; + M_HASHTYPE_SET(rxr->fmp, M_HASHTYPE_OPAQUE); #endif + } sendmp = rxr->fmp; /* Make sure to set M_PKTHDR. */ sendmp->m_flags |= M_PKTHDR; @@ -5004,10 +5205,7 @@ next_desc: /* * Flush any outstanding LRO work */ - while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { - SLIST_REMOVE_HEAD(&lro->lro_active, next); - tcp_lro_flush(lro, queued); - } + tcp_lro_flush_all(lro); if (done != NULL) *done += rxdone; @@ -5336,6 +5534,100 @@ igb_led_func(void *arg, int onoff) IGB_CORE_UNLOCK(adapter); } +static uint64_t +igb_get_vf_counter(if_t ifp, ift_counter cnt) +{ + struct adapter *adapter; + struct e1000_vf_stats *stats; +#ifndef IGB_LEGACY_TX + struct tx_ring *txr; + uint64_t rv; +#endif + + adapter = if_getsoftc(ifp); + stats = (struct e1000_vf_stats *)adapter->stats; + + switch (cnt) { + case IFCOUNTER_IPACKETS: + return (stats->gprc); + case IFCOUNTER_OPACKETS: + return (stats->gptc); + case IFCOUNTER_IBYTES: + return (stats->gorc); + case IFCOUNTER_OBYTES: + return (stats->gotc); + case IFCOUNTER_IMCASTS: + return (stats->mprc); + case IFCOUNTER_IERRORS: + return (adapter->dropped_pkts); + case IFCOUNTER_OERRORS: + return (adapter->watchdog_events); +#ifndef IGB_LEGACY_TX + case IFCOUNTER_OQDROPS: + rv = 0; + txr = adapter->tx_rings; + for (int i = 0; i < adapter->num_queues; i++, txr++) + rv += txr->br->br_drops; + return (rv); +#endif + default: + return (if_get_counter_default(ifp, cnt)); + } +} + +static uint64_t +igb_get_counter(if_t ifp, ift_counter cnt) +{ + struct adapter *adapter; + struct e1000_hw_stats *stats; +#ifndef IGB_LEGACY_TX + struct tx_ring *txr; + uint64_t rv; +#endif + + adapter = if_getsoftc(ifp); + if (adapter->vf_ifp) + return (igb_get_vf_counter(ifp, cnt)); + + stats = (struct e1000_hw_stats *)adapter->stats; + + switch (cnt) { + case IFCOUNTER_IPACKETS: + return (stats->gprc); + case IFCOUNTER_OPACKETS: + return (stats->gptc); + case IFCOUNTER_IBYTES: + return (stats->gorc); + case IFCOUNTER_OBYTES: + return (stats->gotc); + case IFCOUNTER_IMCASTS: + return (stats->mprc); + case IFCOUNTER_OMCASTS: + return (stats->mptc); + case IFCOUNTER_IERRORS: + return (adapter->dropped_pkts + stats->rxerrc + + stats->crcerrs + stats->algnerrc + + stats->ruc + stats->roc + stats->cexterr); + case IFCOUNTER_OERRORS: + return (stats->ecol + stats->latecol + + adapter->watchdog_events); + case IFCOUNTER_COLLISIONS: + return (stats->colc); + case IFCOUNTER_IQDROPS: + return (stats->mpc); +#ifndef IGB_LEGACY_TX + case IFCOUNTER_OQDROPS: + rv = 0; + txr = adapter->tx_rings; + for (int i = 0; i < adapter->num_queues; i++, txr++) + rv += txr->br->br_drops; + return (rv); +#endif + default: + return (if_get_counter_default(ifp, cnt)); + } +} + /********************************************************************** * * Update the board statistics counters. @@ -5344,7 +5636,6 @@ igb_led_func(void *arg, int onoff) static void igb_update_stats_counters(struct adapter *adapter) { - struct ifnet *ifp; struct e1000_hw *hw = &adapter->hw; struct e1000_hw_stats *stats; @@ -5360,7 +5651,7 @@ igb_update_stats_counters(struct adapter *adapter) stats = (struct e1000_hw_stats *)adapter->stats; - if(adapter->hw.phy.media_type == e1000_media_type_copper || + if (adapter->hw.phy.media_type == e1000_media_type_copper || (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) { stats->symerrs += E1000_READ_REG(hw,E1000_SYMERRS); @@ -5468,18 +5759,6 @@ igb_update_stats_counters(struct adapter *adapter) stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC); stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC); - ifp = adapter->ifp; - ifp->if_collisions = stats->colc; - - /* Rx Errors */ - ifp->if_ierrors = adapter->dropped_pkts + stats->rxerrc + - stats->crcerrs + stats->algnerrc + - stats->ruc + stats->roc + stats->mpc + stats->cexterr; - - /* Tx Errors */ - ifp->if_oerrors = stats->ecol + - stats->latecol + adapter->watchdog_events; - /* Driver specific counters */ adapter->device_control = E1000_READ_REG(hw, E1000_CTRL); adapter->rx_control = E1000_READ_REG(hw, E1000_RCTL); @@ -5598,12 +5877,15 @@ igb_add_hw_stats(struct adapter *adapter) char namebuf[QUEUE_NAME_LEN]; /* Driver Statistics */ - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq", - CTLFLAG_RD, &adapter->link_irq, - "Link MSIX IRQ Handled"); SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", CTLFLAG_RD, &adapter->dropped_pkts, "Driver dropped packets"); + SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq", + CTLFLAG_RD, &adapter->link_irq, + "Link MSIX IRQ Handled"); + SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_fail", + CTLFLAG_RD, &adapter->mbuf_defrag_failed, + "Defragmenting mbuf chain failed"); SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail", CTLFLAG_RD, &adapter->no_tx_dma_setup, "Driver tx dma failure in xmit"); @@ -5663,7 +5945,7 @@ igb_add_hw_stats(struct adapter *adapter) "Transmit Descriptor Tail"); SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "no_desc_avail", CTLFLAG_RD, &txr->no_desc_avail, - "Queue No Descriptor Available"); + "Queue Descriptors Unavailable"); SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets", CTLFLAG_RD, &txr->total_packets, "Queue Packets Transmitted"); @@ -5682,10 +5964,10 @@ igb_add_hw_stats(struct adapter *adapter) SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_bytes", CTLFLAG_RD, &rxr->rx_bytes, "Queue Bytes Received"); - SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "lro_queued", + SYSCTL_ADD_U64(ctx, queue_list, OID_AUTO, "lro_queued", CTLFLAG_RD, &lro->lro_queued, 0, "LRO Queued"); - SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "lro_flushed", + SYSCTL_ADD_U64(ctx, queue_list, OID_AUTO, "lro_flushed", CTLFLAG_RD, &lro->lro_flushed, 0, "LRO Flushed"); } @@ -5837,18 +6119,18 @@ igb_add_hw_stats(struct adapter *adapter) "1023-1522 byte frames received"); SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", CTLFLAG_RD, &stats->gorc, - "Good Octets Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_recvd", - CTLFLAG_RD, &stats->tor, - "Total Octets Received"); + "Good Octets Received"); + SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_recvd", + CTLFLAG_RD, &stats->tor, + "Total Octets Received"); /* Packet Transmission Stats */ SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", CTLFLAG_RD, &stats->gotc, "Good Octets Transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_txd", - CTLFLAG_RD, &stats->tot, - "Total Octets Transmitted"); + SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_txd", + CTLFLAG_RD, &stats->tot, + "Total Octets Transmitted"); SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd", CTLFLAG_RD, &stats->tpt, "Total Packets Transmitted"); @@ -6085,6 +6367,7 @@ igb_set_flowcntl(SYSCTL_HANDLER_ARGS) adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode; e1000_force_mac_fc(&adapter->hw); + /* XXX TODO: update DROP_EN on each RX queue if appropriate */ return (error); } @@ -6108,7 +6391,7 @@ igb_sysctl_dmac(SYSCTL_HANDLER_ARGS) switch (adapter->dmac) { case 0: - /*Disabling */ + /* Disabling */ break; case 1: /* Just enable and use default */ adapter->dmac = 1000; diff --git a/freebsd/sys/dev/e1000/if_igb.h b/freebsd/sys/dev/e1000/if_igb.h index 0c447412..ea5ba649 100644 --- a/freebsd/sys/dev/e1000/if_igb.h +++ b/freebsd/sys/dev/e1000/if_igb.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2013, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -32,11 +32,70 @@ ******************************************************************************/ /*$FreeBSD$*/ -#ifndef _IGB_H_DEFINED_ -#define _IGB_H_DEFINED_ +#ifndef _IF_IGB_H_ +#define _IF_IGB_H_ -/* Tunables */ +#ifdef ALTQ +#define IGB_LEGACY_TX +#endif + +#include <rtems/bsd/sys/param.h> +#include <sys/systm.h> +#ifndef IGB_LEGACY_TX +#include <sys/buf_ring.h> +#endif +#include <sys/bus.h> +#include <sys/endian.h> +#include <sys/kernel.h> +#include <sys/kthread.h> +#include <sys/malloc.h> +#include <sys/mbuf.h> +#include <sys/module.h> +#include <sys/rman.h> +#include <sys/socket.h> +#include <sys/sockio.h> +#include <sys/sysctl.h> +#include <sys/taskqueue.h> +#include <sys/eventhandler.h> +#include <sys/pcpu.h> +#include <sys/smp.h> +#include <machine/smp.h> +#include <machine/bus.h> +#include <machine/resource.h> + +#include <net/bpf.h> +#include <net/ethernet.h> +#include <net/if.h> +#include <net/if_var.h> +#include <net/if_arp.h> +#include <net/if_dl.h> +#include <net/if_media.h> +#ifdef RSS +#include <net/rss_config.h> +#include <netinet/in_rss.h> +#endif +#include <net/if_types.h> +#include <net/if_vlan_var.h> + +#include <netinet/in_systm.h> +#include <netinet/in.h> +#include <netinet/if_ether.h> +#include <netinet/ip.h> +#include <netinet/ip6.h> +#include <netinet/tcp.h> +#include <netinet/tcp_lro.h> +#include <netinet/udp.h> + +#include <machine/in_cksum.h> +#include <dev/led/led.h> +#include <dev/pci/pcivar.h> +#include <dev/pci/pcireg.h> + +#include "e1000_api.h" +#include "e1000_82575.h" + +/* Tunables */ /* * IGB_TXD: Maximum number of Transmit Descriptors * @@ -168,7 +227,7 @@ /* * Micellaneous constants */ -#define IGB_VENDOR_ID 0x8086 +#define IGB_INTEL_VENDOR_ID 0x8086 #define IGB_JUMBO_PBA 0x00000028 #define IGB_DEFAULT_PBA 0x00000030 @@ -223,7 +282,7 @@ #define HW_DEBUGOUT1(S, A) if (DEBUG_HW) printf(S "\n", A) #define HW_DEBUGOUT2(S, A, B) if (DEBUG_HW) printf(S "\n", A, B) -#define IGB_MAX_SCATTER 64 +#define IGB_MAX_SCATTER 40 #define IGB_VFTA_SIZE 128 #define IGB_BR_SIZE 4096 /* ring buf size */ #define IGB_TSO_SIZE (65535 + sizeof(struct ether_vlan_header)) @@ -236,7 +295,11 @@ #define ETH_ADDR_LEN 6 /* Offload bits in mbuf flag */ -#if __FreeBSD_version >= 800000 +#if __FreeBSD_version >= 1000000 +#define CSUM_OFFLOAD_IPV4 (CSUM_IP|CSUM_IP_TCP|CSUM_IP_UDP|CSUM_IP_SCTP) +#define CSUM_OFFLOAD_IPV6 (CSUM_IP6_TCP|CSUM_IP6_UDP|CSUM_IP6_SCTP) +#define CSUM_OFFLOAD (CSUM_OFFLOAD_IPV4|CSUM_OFFLOAD_IPV6) +#elif __FreeBSD_version >= 800000 #define CSUM_OFFLOAD (CSUM_IP|CSUM_TCP|CSUM_UDP|CSUM_SCTP) #else #define CSUM_OFFLOAD (CSUM_IP|CSUM_TCP|CSUM_UDP) @@ -300,7 +363,6 @@ struct tx_ring { volatile u16 tx_avail; u16 next_avail_desc; u16 next_to_clean; - u16 process_limit; u16 num_desc; enum { IGB_QUEUE_IDLE = 1, @@ -336,7 +398,6 @@ struct rx_ring { struct lro_ctrl lro; bool lro_enabled; bool hdr_split; - bool discard; struct mtx rx_mtx; char mtx_name[16]; u32 next_to_refresh; @@ -368,7 +429,7 @@ struct adapter { struct e1000_hw hw; struct e1000_osdep osdep; - struct device *dev; + device_t dev; struct cdev *led_dev; struct resource *pci_mem; @@ -459,20 +520,19 @@ struct adapter { u8 *mta; /* Misc stats maintained by the driver */ + unsigned long device_control; unsigned long dropped_pkts; + unsigned long eint_mask; + unsigned long int_mask; + unsigned long link_irq; unsigned long mbuf_defrag_failed; - unsigned long mbuf_header_failed; - unsigned long mbuf_packet_failed; unsigned long no_tx_dma_setup; - unsigned long watchdog_events; - unsigned long link_irq; - unsigned long rx_overruns; - unsigned long device_control; - unsigned long rx_control; - unsigned long int_mask; - unsigned long eint_mask; unsigned long packet_buf_alloc_rx; unsigned long packet_buf_alloc_tx; + unsigned long rx_control; + unsigned long rx_overruns; + unsigned long watchdog_events; + /* Used in pf and vf */ void *stats; @@ -480,6 +540,7 @@ struct adapter { int has_manage; int wol; int rx_process_limit; + int tx_process_limit; u16 vf_ifp; /* a VF interface */ bool in_detach; /* Used only in igb_ioctl */ @@ -568,6 +629,6 @@ drbr_needs_enqueue(struct ifnet *ifp, struct buf_ring *br) } #endif -#endif /* _IGB_H_DEFINED_ */ +#endif /* _IF_IGB_H_ */ diff --git a/freebsd/sys/dev/e1000/if_lem.c b/freebsd/sys/dev/e1000/if_lem.c index 7c22200d..c46c3728 100644 --- a/freebsd/sys/dev/e1000/if_lem.c +++ b/freebsd/sys/dev/e1000/if_lem.c @@ -2,7 +2,7 @@ /****************************************************************************** - Copyright (c) 2001-2012, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -34,6 +34,15 @@ ******************************************************************************/ /*$FreeBSD$*/ +/* + * Uncomment the following extensions for better performance in a VM, + * especially if you have support in the hypervisor. + * See http://info.iet.unipi.it/~luigi/netmap/ + */ +// #define BATCH_DISPATCH +// #define NIC_SEND_COMBINING +// #define NIC_PARAVIRT /* enable virtio-like synchronization */ + #include <rtems/bsd/local/opt_inet.h> #include <rtems/bsd/local/opt_inet6.h> @@ -43,6 +52,7 @@ #include <rtems/bsd/sys/param.h> #include <sys/systm.h> +#include <sys/buf_ring.h> #include <sys/bus.h> #include <sys/endian.h> #include <sys/kernel.h> @@ -62,6 +72,7 @@ #include <net/bpf.h> #include <net/ethernet.h> #include <net/if.h> +#include <net/if_var.h> #include <net/if_arp.h> #include <net/if_dl.h> #include <net/if_media.h> @@ -88,7 +99,7 @@ /********************************************************************* * Legacy Em Driver version: *********************************************************************/ -char lem_driver_version[] = "1.0.6"; +char lem_driver_version[] = "1.1.0"; /********************************************************************* * PCI Device ID Table @@ -168,14 +179,15 @@ static int lem_detach(device_t); static int lem_shutdown(device_t); static int lem_suspend(device_t); static int lem_resume(device_t); -static void lem_start(struct ifnet *); -static void lem_start_locked(struct ifnet *ifp); -static int lem_ioctl(struct ifnet *, u_long, caddr_t); +static void lem_start(if_t); +static void lem_start_locked(if_t ifp); +static int lem_ioctl(if_t, u_long, caddr_t); +static uint64_t lem_get_counter(if_t, ift_counter); static void lem_init(void *); static void lem_init_locked(struct adapter *); static void lem_stop(void *); -static void lem_media_status(struct ifnet *, struct ifmediareq *); -static int lem_media_change(struct ifnet *); +static void lem_media_status(if_t, struct ifmediareq *); +static int lem_media_change(if_t); static void lem_identify_hardware(struct adapter *); static int lem_allocate_pci_resources(struct adapter *); static int lem_allocate_irq(struct adapter *adapter); @@ -210,8 +222,8 @@ static void lem_disable_promisc(struct adapter *); static void lem_set_multi(struct adapter *); static void lem_update_link_status(struct adapter *); static int lem_get_buf(struct adapter *, int); -static void lem_register_vlan(void *, struct ifnet *, u16); -static void lem_unregister_vlan(void *, struct ifnet *, u16); +static void lem_register_vlan(void *, if_t, u16); +static void lem_unregister_vlan(void *, if_t, u16); static void lem_setup_vlan_hw_support(struct adapter *); static int lem_xmit(struct adapter *, struct mbuf **); static void lem_smartspeed(struct adapter *); @@ -276,6 +288,9 @@ extern devclass_t em_devclass; DRIVER_MODULE(lem, pci, lem_driver, em_devclass, 0, 0); MODULE_DEPEND(lem, pci, 1, 1, 1); MODULE_DEPEND(lem, ether, 1, 1, 1); +#ifdef DEV_NETMAP +MODULE_DEPEND(lem, netmap, 1, 1, 1); +#endif /* DEV_NETMAP */ /********************************************************************* * Tunable default values. @@ -291,6 +306,10 @@ static int lem_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV); static int lem_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR); static int lem_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV); static int lem_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV); +/* + * increase lem_rxd and lem_txd to at least 2048 in netmap mode + * for better performance. + */ static int lem_rxd = EM_DEFAULT_RXD; static int lem_txd = EM_DEFAULT_TXD; static int lem_smart_pwr_down = FALSE; @@ -460,6 +479,20 @@ lem_attach(device_t dev) "max number of rx packets to process", &adapter->rx_process_limit, lem_rx_process_limit); +#ifdef NIC_SEND_COMBINING + /* Sysctls to control mitigation */ + lem_add_rx_process_limit(adapter, "sc_enable", + "driver TDT mitigation", &adapter->sc_enable, 0); +#endif /* NIC_SEND_COMBINING */ +#ifdef BATCH_DISPATCH + lem_add_rx_process_limit(adapter, "batch_enable", + "driver rx batch", &adapter->batch_enable, 0); +#endif /* BATCH_DISPATCH */ +#ifdef NIC_PARAVIRT + lem_add_rx_process_limit(adapter, "rx_retries", + "driver rx retries", &adapter->rx_retries, 0); +#endif /* NIC_PARAVIRT */ + /* Sysctl for setting the interface flow control */ lem_set_flow_cntrl(adapter, "flow_control", "flow control setting", @@ -517,6 +550,49 @@ lem_attach(device_t dev) */ adapter->hw.mac.report_tx_early = 1; +#ifdef NIC_PARAVIRT + device_printf(dev, "driver supports paravirt, subdev 0x%x\n", + adapter->hw.subsystem_device_id); + if (adapter->hw.subsystem_device_id == E1000_PARA_SUBDEV) { + uint64_t bus_addr; + + device_printf(dev, "paravirt support on dev %p\n", adapter); + tsize = 4096; // XXX one page for the csb + if (lem_dma_malloc(adapter, tsize, &adapter->csb_mem, BUS_DMA_NOWAIT)) { + device_printf(dev, "Unable to allocate csb memory\n"); + error = ENOMEM; + goto err_csb; + } + /* Setup the Base of the CSB */ + adapter->csb = (struct paravirt_csb *)adapter->csb_mem.dma_vaddr; + /* force the first kick */ + adapter->csb->host_need_txkick = 1; /* txring empty */ + adapter->csb->guest_need_rxkick = 1; /* no rx packets */ + bus_addr = adapter->csb_mem.dma_paddr; + lem_add_rx_process_limit(adapter, "csb_on", + "enable paravirt.", &adapter->csb->guest_csb_on, 0); + lem_add_rx_process_limit(adapter, "txc_lim", + "txc_lim", &adapter->csb->host_txcycles_lim, 1); + + /* some stats */ +#define PA_SC(name, var, val) \ + lem_add_rx_process_limit(adapter, name, name, var, val) + PA_SC("host_need_txkick",&adapter->csb->host_need_txkick, 1); + PA_SC("host_rxkick_at",&adapter->csb->host_rxkick_at, ~0); + PA_SC("guest_need_txkick",&adapter->csb->guest_need_txkick, 0); + PA_SC("guest_need_rxkick",&adapter->csb->guest_need_rxkick, 1); + PA_SC("tdt_reg_count",&adapter->tdt_reg_count, 0); + PA_SC("tdt_csb_count",&adapter->tdt_csb_count, 0); + PA_SC("tdt_int_count",&adapter->tdt_int_count, 0); + PA_SC("guest_need_kick_count",&adapter->guest_need_kick_count, 0); + /* tell the host where the block is */ + E1000_WRITE_REG(&adapter->hw, E1000_CSBAH, + (u32)(bus_addr >> 32)); + E1000_WRITE_REG(&adapter->hw, E1000_CSBAL, + (u32)bus_addr); + } +#endif /* NIC_PARAVIRT */ + tsize = roundup2(adapter->num_tx_desc * sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); @@ -654,7 +730,7 @@ lem_attach(device_t dev) lem_get_hw_control(adapter); /* Tell the stack that the interface is not active */ - adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); + if_setdrvflagbits(adapter->ifp, 0, IFF_DRV_OACTIVE | IFF_DRV_RUNNING); adapter->led_dev = led_create(lem_led_func, adapter, device_get_nameunit(dev)); @@ -675,8 +751,13 @@ err_hw_init: err_rx_desc: lem_dma_free(adapter, &adapter->txdma); err_tx_desc: +#ifdef NIC_PARAVIRT + lem_dma_free(adapter, &adapter->csb_mem); +err_csb: +#endif /* NIC_PARAVIRT */ + err_pci: - if (adapter->ifp != NULL) + if (adapter->ifp != (void *)NULL) if_free(adapter->ifp); lem_free_pci_resources(adapter); free(adapter->mta, M_DEVBUF); @@ -701,18 +782,18 @@ static int lem_detach(device_t dev) { struct adapter *adapter = device_get_softc(dev); - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; INIT_DEBUGOUT("em_detach: begin"); /* Make sure VLANS are not using driver */ - if (adapter->ifp->if_vlantrunk != NULL) { + if (if_vlantrunkinuse(ifp)) { device_printf(dev,"Vlan in use, detach first\n"); return (EBUSY); } #ifdef DEVICE_POLLING - if (ifp->if_capenable & IFCAP_POLLING) + if (if_getcapenable(ifp) & IFCAP_POLLING) ether_poll_deregister(ifp); #endif @@ -762,6 +843,12 @@ lem_detach(device_t dev) adapter->rx_desc_base = NULL; } +#ifdef NIC_PARAVIRT + if (adapter->csb) { + lem_dma_free(adapter, &adapter->csb_mem); + adapter->csb = NULL; + } +#endif /* NIC_PARAVIRT */ lem_release_hw_control(adapter); free(adapter->mta, M_DEVBUF); EM_TX_LOCK_DESTROY(adapter); @@ -806,7 +893,7 @@ static int lem_resume(device_t dev) { struct adapter *adapter = device_get_softc(dev); - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; EM_CORE_LOCK(adapter); lem_init_locked(adapter); @@ -819,14 +906,14 @@ lem_resume(device_t dev) static void -lem_start_locked(struct ifnet *ifp) +lem_start_locked(if_t ifp) { - struct adapter *adapter = ifp->if_softc; + struct adapter *adapter = if_getsoftc(ifp); struct mbuf *m_head; EM_TX_LOCK_ASSERT(adapter); - if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != + if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != IFF_DRV_RUNNING) return; if (!adapter->link_active) @@ -845,9 +932,9 @@ lem_start_locked(struct ifnet *ifp) } } - while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { + while (!if_sendq_empty(ifp)) { + m_head = if_dequeue(ifp); - IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); if (m_head == NULL) break; /* @@ -857,31 +944,41 @@ lem_start_locked(struct ifnet *ifp) if (lem_xmit(adapter, &m_head)) { if (m_head == NULL) break; - ifp->if_drv_flags |= IFF_DRV_OACTIVE; - IFQ_DRV_PREPEND(&ifp->if_snd, m_head); + if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0); + if_sendq_prepend(ifp, m_head); break; } /* Send a copy of the frame to the BPF listener */ - ETHER_BPF_MTAP(ifp, m_head); + if_etherbpfmtap(ifp, m_head); /* Set timeout in case hardware has problems transmitting. */ adapter->watchdog_check = TRUE; adapter->watchdog_time = ticks; } if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) - ifp->if_drv_flags |= IFF_DRV_OACTIVE; + if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0); +#ifdef NIC_PARAVIRT + if (if_getdrvflags(ifp) & IFF_DRV_OACTIVE && adapter->csb && + adapter->csb->guest_csb_on && + !(adapter->csb->guest_need_txkick & 1)) { + adapter->csb->guest_need_txkick = 1; + adapter->guest_need_kick_count++; + // XXX memory barrier + lem_txeof(adapter); // XXX possibly clear IFF_DRV_OACTIVE + } +#endif /* NIC_PARAVIRT */ return; } static void -lem_start(struct ifnet *ifp) +lem_start(if_t ifp) { - struct adapter *adapter = ifp->if_softc; + struct adapter *adapter = if_getsoftc(ifp); EM_TX_LOCK(adapter); - if (ifp->if_drv_flags & IFF_DRV_RUNNING) + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) lem_start_locked(ifp); EM_TX_UNLOCK(adapter); } @@ -896,9 +993,9 @@ lem_start(struct ifnet *ifp) **********************************************************************/ static int -lem_ioctl(struct ifnet *ifp, u_long command, caddr_t data) +lem_ioctl(if_t ifp, u_long command, caddr_t data) { - struct adapter *adapter = ifp->if_softc; + struct adapter *adapter = if_getsoftc(ifp); struct ifreq *ifr = (struct ifreq *)data; #if defined(INET) || defined(INET6) struct ifaddr *ifa = (struct ifaddr *)data; @@ -924,11 +1021,11 @@ lem_ioctl(struct ifnet *ifp, u_long command, caddr_t data) ** so we avoid doing it when possible. */ if (avoid_reset) { - ifp->if_flags |= IFF_UP; - if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) + if_setflagbits(ifp, IFF_UP, 0); + if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) lem_init(adapter); #ifdef INET - if (!(ifp->if_flags & IFF_NOARP)) + if (!(if_getflags(ifp) & IFF_NOARP)) arp_ifinit(ifp, ifa); #endif } else @@ -955,10 +1052,11 @@ lem_ioctl(struct ifnet *ifp, u_long command, caddr_t data) break; } - ifp->if_mtu = ifr->ifr_mtu; + if_setmtu(ifp, ifr->ifr_mtu); adapter->max_frame_size = - ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; - lem_init_locked(adapter); + if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN; + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) + lem_init_locked(adapter); EM_CORE_UNLOCK(adapter); break; } @@ -966,9 +1064,9 @@ lem_ioctl(struct ifnet *ifp, u_long command, caddr_t data) IOCTL_DEBUGOUT("ioctl rcv'd:\ SIOCSIFFLAGS (Set Interface Flags)"); EM_CORE_LOCK(adapter); - if (ifp->if_flags & IFF_UP) { - if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { - if ((ifp->if_flags ^ adapter->if_flags) & + if (if_getflags(ifp) & IFF_UP) { + if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING)) { + if ((if_getflags(ifp) ^ adapter->if_flags) & (IFF_PROMISC | IFF_ALLMULTI)) { lem_disable_promisc(adapter); lem_set_promisc(adapter); @@ -976,18 +1074,18 @@ lem_ioctl(struct ifnet *ifp, u_long command, caddr_t data) } else lem_init_locked(adapter); } else - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { EM_TX_LOCK(adapter); lem_stop(adapter); EM_TX_UNLOCK(adapter); } - adapter->if_flags = ifp->if_flags; + adapter->if_flags = if_getflags(ifp); EM_CORE_UNLOCK(adapter); break; case SIOCADDMULTI: case SIOCDELMULTI: IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { EM_CORE_LOCK(adapter); lem_disable_intr(adapter); lem_set_multi(adapter); @@ -996,7 +1094,7 @@ lem_ioctl(struct ifnet *ifp, u_long command, caddr_t data) lem_initialize_receive_unit(adapter); } #ifdef DEVICE_POLLING - if (!(ifp->if_capenable & IFCAP_POLLING)) + if (!(if_getcapenable(ifp) & IFCAP_POLLING)) #endif lem_enable_intr(adapter); EM_CORE_UNLOCK(adapter); @@ -1023,7 +1121,7 @@ lem_ioctl(struct ifnet *ifp, u_long command, caddr_t data) IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); reinit = 0; - mask = ifr->ifr_reqcap ^ ifp->if_capenable; + mask = ifr->ifr_reqcap ^ if_getcapenable(ifp); #ifdef DEVICE_POLLING if (mask & IFCAP_POLLING) { if (ifr->ifr_reqcap & IFCAP_POLLING) { @@ -1032,36 +1130,36 @@ lem_ioctl(struct ifnet *ifp, u_long command, caddr_t data) return (error); EM_CORE_LOCK(adapter); lem_disable_intr(adapter); - ifp->if_capenable |= IFCAP_POLLING; + if_setcapenablebit(ifp, IFCAP_POLLING, 0); EM_CORE_UNLOCK(adapter); } else { error = ether_poll_deregister(ifp); /* Enable interrupt even in error case */ EM_CORE_LOCK(adapter); lem_enable_intr(adapter); - ifp->if_capenable &= ~IFCAP_POLLING; + if_setcapenablebit(ifp, 0, IFCAP_POLLING); EM_CORE_UNLOCK(adapter); } } #endif if (mask & IFCAP_HWCSUM) { - ifp->if_capenable ^= IFCAP_HWCSUM; + if_togglecapenable(ifp, IFCAP_HWCSUM); reinit = 1; } if (mask & IFCAP_VLAN_HWTAGGING) { - ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; + if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING); reinit = 1; } if ((mask & IFCAP_WOL) && - (ifp->if_capabilities & IFCAP_WOL) != 0) { + (if_getcapabilities(ifp) & IFCAP_WOL) != 0) { if (mask & IFCAP_WOL_MCAST) - ifp->if_capenable ^= IFCAP_WOL_MCAST; + if_togglecapenable(ifp, IFCAP_WOL_MCAST); if (mask & IFCAP_WOL_MAGIC) - ifp->if_capenable ^= IFCAP_WOL_MAGIC; + if_togglecapenable(ifp, IFCAP_WOL_MAGIC); } - if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) + if (reinit && (if_getdrvflags(ifp) & IFF_DRV_RUNNING)) lem_init(adapter); - VLAN_CAPABILITIES(ifp); + if_vlancap(ifp); break; } @@ -1088,7 +1186,7 @@ lem_ioctl(struct ifnet *ifp, u_long command, caddr_t data) static void lem_init_locked(struct adapter *adapter) { - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; device_t dev = adapter->dev; u32 pba; @@ -1135,7 +1233,7 @@ lem_init_locked(struct adapter *adapter) E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba); /* Get the latest mac address, User can use a LAA */ - bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr, + bcopy(if_getlladdr(adapter->ifp), adapter->hw.mac.addr, ETHER_ADDR_LEN); /* Put the address into the Receive Address Array */ @@ -1152,10 +1250,10 @@ lem_init_locked(struct adapter *adapter) E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); /* Set hardware offload abilities */ - ifp->if_hwassist = 0; + if_clearhwassist(ifp); if (adapter->hw.mac.type >= e1000_82543) { - if (ifp->if_capenable & IFCAP_TXCSUM) - ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); + if (if_getcapenable(ifp) & IFCAP_TXCSUM) + if_sethwassistbits(ifp, CSUM_TCP | CSUM_UDP, 0); } /* Configure for OS presence */ @@ -1179,8 +1277,8 @@ lem_init_locked(struct adapter *adapter) lem_initialize_receive_unit(adapter); /* Use real VLAN Filter support? */ - if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) { - if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) + if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) { + if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) /* Use real VLAN Filter support */ lem_setup_vlan_hw_support(adapter); else { @@ -1194,8 +1292,7 @@ lem_init_locked(struct adapter *adapter) /* Don't lose promiscuous settings */ lem_set_promisc(adapter); - ifp->if_drv_flags |= IFF_DRV_RUNNING; - ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; + if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); callout_reset(&adapter->timer, hz, lem_local_timer, adapter); e1000_clear_hw_cntrs_base_generic(&adapter->hw); @@ -1205,7 +1302,7 @@ lem_init_locked(struct adapter *adapter) * Only enable interrupts if we are not polling, make sure * they are off otherwise. */ - if (ifp->if_capenable & IFCAP_POLLING) + if (if_getcapenable(ifp) & IFCAP_POLLING) lem_disable_intr(adapter); else #endif /* DEVICE_POLLING */ @@ -1234,13 +1331,13 @@ lem_init(void *arg) * *********************************************************************/ static int -lem_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) +lem_poll(if_t ifp, enum poll_cmd cmd, int count) { - struct adapter *adapter = ifp->if_softc; + struct adapter *adapter = if_getsoftc(ifp); u32 reg_icr, rx_done = 0; EM_CORE_LOCK(adapter); - if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { + if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) { EM_CORE_UNLOCK(adapter); return (rx_done); } @@ -1261,7 +1358,7 @@ lem_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) EM_TX_LOCK(adapter); lem_txeof(adapter); - if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) + if(!if_sendq_empty(ifp)) lem_start_locked(ifp); EM_TX_UNLOCK(adapter); return (rx_done); @@ -1277,12 +1374,12 @@ static void lem_intr(void *arg) { struct adapter *adapter = arg; - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; u32 reg_icr; - if ((ifp->if_capenable & IFCAP_POLLING) || - ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)) + if ((if_getcapenable(ifp) & IFCAP_POLLING) || + ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)) return; EM_CORE_LOCK(adapter); @@ -1312,8 +1409,8 @@ lem_intr(void *arg) EM_TX_LOCK(adapter); lem_txeof(adapter); - if (ifp->if_drv_flags & IFF_DRV_RUNNING && - !IFQ_DRV_IS_EMPTY(&ifp->if_snd)) + if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) && + (!if_sendq_empty(ifp))) lem_start_locked(ifp); EM_TX_UNLOCK(adapter); return; @@ -1324,9 +1421,9 @@ static void lem_handle_link(void *context, int pending) { struct adapter *adapter = context; - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; - if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) + if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) return; EM_CORE_LOCK(adapter); @@ -1344,14 +1441,14 @@ static void lem_handle_rxtx(void *context, int pending) { struct adapter *adapter = context; - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { bool more = lem_rxeof(adapter, adapter->rx_process_limit, NULL); EM_TX_LOCK(adapter); lem_txeof(adapter); - if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) + if(!if_sendq_empty(ifp)) lem_start_locked(ifp); EM_TX_UNLOCK(adapter); if (more) { @@ -1360,7 +1457,7 @@ lem_handle_rxtx(void *context, int pending) } } - if (ifp->if_drv_flags & IFF_DRV_RUNNING) + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) lem_enable_intr(adapter); } @@ -1373,7 +1470,7 @@ static int lem_irq_fast(void *arg) { struct adapter *adapter = arg; - struct ifnet *ifp; + if_t ifp; u32 reg_icr; ifp = adapter->ifp; @@ -1417,9 +1514,9 @@ lem_irq_fast(void *arg) * **********************************************************************/ static void -lem_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) +lem_media_status(if_t ifp, struct ifmediareq *ifmr) { - struct adapter *adapter = ifp->if_softc; + struct adapter *adapter = if_getsoftc(ifp); u_char fiber_type = IFM_1000_SX; INIT_DEBUGOUT("lem_media_status: begin"); @@ -1471,9 +1568,9 @@ lem_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) * **********************************************************************/ static int -lem_media_change(struct ifnet *ifp) +lem_media_change(if_t ifp) { - struct adapter *adapter = ifp->if_softc; + struct adapter *adapter = if_getsoftc(ifp); struct ifmedia *ifm = &adapter->media; INIT_DEBUGOUT("lem_media_change: begin"); @@ -1581,9 +1678,9 @@ lem_xmit(struct adapter *adapter, struct mbuf **m_headp) if (error == EFBIG) { struct mbuf *m; - m = m_defrag(*m_headp, M_NOWAIT); + m = m_collapse(*m_headp, M_NOWAIT, EM_MAX_SCATTER); if (m == NULL) { - adapter->mbuf_alloc_failed++; + adapter->mbuf_defrag_failed++; m_freem(*m_headp); *m_headp = NULL; return (ENOBUFS); @@ -1605,7 +1702,7 @@ lem_xmit(struct adapter *adapter, struct mbuf **m_headp) return (error); } - if (nsegs > (adapter->num_tx_desc_avail - 2)) { + if (adapter->num_tx_desc_avail < (nsegs + 2)) { adapter->no_tx_desc_avail2++; bus_dmamap_unload(adapter->txtag, map); return (ENOBUFS); @@ -1717,6 +1814,37 @@ lem_xmit(struct adapter *adapter, struct mbuf **m_headp) */ bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + +#ifdef NIC_PARAVIRT + if (adapter->csb) { + adapter->csb->guest_tdt = i; + /* XXX memory barrier ? */ + if (adapter->csb->guest_csb_on && + !(adapter->csb->host_need_txkick & 1)) { + /* XXX maybe useless + * clean the ring. maybe do it before ? + * maybe a little bit of histeresys ? + */ + if (adapter->num_tx_desc_avail <= 64) {// XXX + lem_txeof(adapter); + } + return (0); + } + } +#endif /* NIC_PARAVIRT */ + +#ifdef NIC_SEND_COMBINING + if (adapter->sc_enable) { + if (adapter->shadow_tdt & MIT_PENDING_INT) { + /* signal intr and data pending */ + adapter->shadow_tdt = MIT_PENDING_TDT | (i & 0xffff); + return (0); + } else { + adapter->shadow_tdt = MIT_PENDING_INT; + } + } +#endif /* NIC_SEND_COMBINING */ + if (adapter->hw.mac.type == e1000_82547 && adapter->link_duplex == HALF_DUPLEX) lem_82547_move_tail(adapter); @@ -1850,18 +1978,18 @@ lem_82547_tx_fifo_reset(struct adapter *adapter) static void lem_set_promisc(struct adapter *adapter) { - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; u32 reg_rctl; reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); - if (ifp->if_flags & IFF_PROMISC) { + if (if_getflags(ifp) & IFF_PROMISC) { reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); /* Turn this on if you want to see bad packets */ if (lem_debug_sbp) reg_rctl |= E1000_RCTL_SBP; E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); - } else if (ifp->if_flags & IFF_ALLMULTI) { + } else if (if_getflags(ifp) & IFF_ALLMULTI) { reg_rctl |= E1000_RCTL_MPE; reg_rctl &= ~E1000_RCTL_UPE; E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); @@ -1871,34 +1999,17 @@ lem_set_promisc(struct adapter *adapter) static void lem_disable_promisc(struct adapter *adapter) { - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; u32 reg_rctl; int mcnt = 0; reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); reg_rctl &= (~E1000_RCTL_UPE); - if (ifp->if_flags & IFF_ALLMULTI) + if (if_getflags(ifp) & IFF_ALLMULTI) mcnt = MAX_NUM_MULTICAST_ADDRESSES; - else { - struct ifmultiaddr *ifma; -#if __FreeBSD_version < 800000 - IF_ADDR_LOCK(ifp); -#else - if_maddr_rlock(ifp); -#endif - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { - if (ifma->ifma_addr->sa_family != AF_LINK) - continue; - if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) - break; - mcnt++; - } -#if __FreeBSD_version < 800000 - IF_ADDR_UNLOCK(ifp); -#else - if_maddr_runlock(ifp); -#endif - } + else + mcnt = if_multiaddr_count(ifp, MAX_NUM_MULTICAST_ADDRESSES); + /* Don't disable if in MAX groups */ if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) reg_rctl &= (~E1000_RCTL_MPE); @@ -1917,8 +2028,7 @@ lem_disable_promisc(struct adapter *adapter) static void lem_set_multi(struct adapter *adapter) { - struct ifnet *ifp = adapter->ifp; - struct ifmultiaddr *ifma; + if_t ifp = adapter->ifp; u32 reg_rctl = 0; u8 *mta; /* Multicast array memory */ int mcnt = 0; @@ -1938,27 +2048,8 @@ lem_set_multi(struct adapter *adapter) msec_delay(5); } -#if __FreeBSD_version < 800000 - IF_ADDR_LOCK(ifp); -#else - if_maddr_rlock(ifp); -#endif - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { - if (ifma->ifma_addr->sa_family != AF_LINK) - continue; - - if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) - break; + if_multiaddr_array(ifp, mta, &mcnt, MAX_NUM_MULTICAST_ADDRESSES); - bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), - &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN); - mcnt++; - } -#if __FreeBSD_version < 800000 - IF_ADDR_UNLOCK(ifp); -#else - if_maddr_runlock(ifp); -#endif if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) { reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); reg_rctl |= E1000_RCTL_MPE; @@ -1997,6 +2088,20 @@ lem_local_timer(void *arg) lem_smartspeed(adapter); +#ifdef NIC_PARAVIRT + /* recover space if needed */ + if (adapter->csb && adapter->csb->guest_csb_on && + (adapter->watchdog_check == TRUE) && + (ticks - adapter->watchdog_time > EM_WATCHDOG) && + (adapter->num_tx_desc_avail != adapter->num_tx_desc) ) { + lem_txeof(adapter); + /* + * lem_txeof() normally (except when space in the queue + * runs low XXX) cleans watchdog_check so that + * we do not hung. + */ + } +#endif /* NIC_PARAVIRT */ /* * We check the watchdog: the time since * the last TX descriptor was cleaned. @@ -2010,7 +2115,7 @@ lem_local_timer(void *arg) return; hung: device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); - adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; + if_setdrvflagbits(adapter->ifp, 0, IFF_DRV_RUNNING); adapter->watchdog_events++; lem_init_locked(adapter); } @@ -2019,7 +2124,7 @@ static void lem_update_link_status(struct adapter *adapter) { struct e1000_hw *hw = &adapter->hw; - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; device_t dev = adapter->dev; u32 link_check = 0; @@ -2060,10 +2165,11 @@ lem_update_link_status(struct adapter *adapter) "Full Duplex" : "Half Duplex")); adapter->link_active = 1; adapter->smartspeed = 0; - ifp->if_baudrate = adapter->link_speed * 1000000; + if_setbaudrate(ifp, adapter->link_speed * 1000000); if_link_state_change(ifp, LINK_STATE_UP); } else if (!link_check && (adapter->link_active == 1)) { - ifp->if_baudrate = adapter->link_speed = 0; + if_setbaudrate(ifp, 0); + adapter->link_speed = 0; adapter->link_duplex = 0; if (bootverbose) device_printf(dev, "Link is Down\n"); @@ -2087,7 +2193,7 @@ static void lem_stop(void *arg) { struct adapter *adapter = arg; - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; EM_CORE_LOCK_ASSERT(adapter); EM_TX_LOCK_ASSERT(adapter); @@ -2099,7 +2205,7 @@ lem_stop(void *arg) callout_stop(&adapter->tx_fifo_timer); /* Tell the stack that the interface is no longer active */ - ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); + if_setdrvflagbits(ifp, 0, (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)); e1000_reset_hw(&adapter->hw); if (adapter->hw.mac.type >= e1000_82544) @@ -2309,7 +2415,7 @@ lem_hardware_init(struct adapter *adapter) * received after sending an XOFF. * - Low water mark works best when it is very near the high water mark. * This allows the receiver to restart by sending XON when it has - * drained a bit. Here we use an arbitary value of 1500 which will + * drained a bit. Here we use an arbitrary value of 1500 which will * restart after one full frame is pulled from the buffer. There * could be several smaller frames in the buffer and if so they will * not trigger the XON until their total number reduces the buffer @@ -2350,40 +2456,40 @@ lem_hardware_init(struct adapter *adapter) static int lem_setup_interface(device_t dev, struct adapter *adapter) { - struct ifnet *ifp; + if_t ifp; INIT_DEBUGOUT("lem_setup_interface: begin"); - ifp = adapter->ifp = if_alloc(IFT_ETHER); - if (ifp == NULL) { + ifp = adapter->ifp = if_gethandle(IFT_ETHER); + if (ifp == (void *)NULL) { device_printf(dev, "can not allocate ifnet structure\n"); return (-1); } if_initname(ifp, device_get_name(dev), device_get_unit(dev)); - ifp->if_init = lem_init; - ifp->if_softc = adapter; - ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; - ifp->if_ioctl = lem_ioctl; - ifp->if_start = lem_start; - IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1); - ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1; - IFQ_SET_READY(&ifp->if_snd); + if_setinitfn(ifp, lem_init); + if_setsoftc(ifp, adapter); + if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); + if_setioctlfn(ifp, lem_ioctl); + if_setstartfn(ifp, lem_start); + if_setgetcounterfn(ifp, lem_get_counter); + if_setsendqlen(ifp, adapter->num_tx_desc - 1); + if_setsendqready(ifp); ether_ifattach(ifp, adapter->hw.mac.addr); - ifp->if_capabilities = ifp->if_capenable = 0; + if_setcapabilities(ifp, 0); if (adapter->hw.mac.type >= e1000_82543) { - ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; - ifp->if_capenable |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; + if_setcapabilitiesbit(ifp, IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM, 0); + if_setcapenablebit(ifp, IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM, 0); } /* * Tell the upper layer(s) we support long frames. */ - ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); - ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; - ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; + if_setifheaderlen(ifp, sizeof(struct ether_vlan_header)); + if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU, 0); + if_setcapenablebit(ifp, IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU, 0); /* ** Dont turn this on by default, if vlans are @@ -2393,16 +2499,16 @@ lem_setup_interface(device_t dev, struct adapter *adapter) ** using vlans directly on the em driver you can ** enable this and get full hardware tag filtering. */ - ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; + if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWFILTER, 0); #ifdef DEVICE_POLLING - ifp->if_capabilities |= IFCAP_POLLING; + if_setcapabilitiesbit(ifp, IFCAP_POLLING, 0); #endif /* Enable only WOL MAGIC by default */ if (adapter->wol) { - ifp->if_capabilities |= IFCAP_WOL; - ifp->if_capenable |= IFCAP_WOL_MAGIC; + if_setcapabilitiesbit(ifp, IFCAP_WOL, 0); + if_setcapenablebit(ifp, IFCAP_WOL_MAGIC, 0); } /* @@ -2565,7 +2671,6 @@ fail_2: bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); bus_dma_tag_destroy(dma->dma_tag); fail_0: - dma->dma_map = NULL; dma->dma_tag = NULL; return (error); @@ -2576,12 +2681,15 @@ lem_dma_free(struct adapter *adapter, struct em_dma_alloc *dma) { if (dma->dma_tag == NULL) return; - if (dma->dma_map != NULL) { + if (dma->dma_paddr != 0) { bus_dmamap_sync(dma->dma_tag, dma->dma_map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(dma->dma_tag, dma->dma_map); + dma->dma_paddr = 0; + } + if (dma->dma_vaddr != NULL) { bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); - dma->dma_map = NULL; + dma->dma_vaddr = NULL; } bus_dma_tag_destroy(dma->dma_tag); dma->dma_tag = NULL; @@ -2656,7 +2764,7 @@ lem_setup_transmit_structures(struct adapter *adapter) struct em_buffer *tx_buffer; #ifdef DEV_NETMAP /* we are already locked */ - struct netmap_adapter *na = NA(adapter->ifp); + struct netmap_adapter *na = netmap_getna(adapter->ifp); struct netmap_slot *slot = netmap_reset(na, NR_TX, 0, 0); #endif /* DEV_NETMAP */ @@ -2679,10 +2787,10 @@ lem_setup_transmit_structures(struct adapter *adapter) uint64_t paddr; void *addr; - addr = PNMB(slot + si, &paddr); + addr = PNMB(na, slot + si, &paddr); adapter->tx_desc_base[i].buffer_addr = htole64(paddr); /* reload the map for netmap mode */ - netmap_load_map(adapter->txtag, tx_buffer->map, addr); + netmap_load_map(na, adapter->txtag, tx_buffer->map, addr); } #endif /* DEV_NETMAP */ tx_buffer->next_eop = -1; @@ -2808,10 +2916,6 @@ lem_free_transmit_structures(struct adapter *adapter) bus_dma_tag_destroy(adapter->txtag); adapter->txtag = NULL; } -#if __FreeBSD_version >= 800000 - if (adapter->br != NULL) - buf_ring_free(adapter->br, M_DEVBUF); -#endif } /********************************************************************* @@ -2982,7 +3086,7 @@ lem_txeof(struct adapter *adapter) int first, last, done, num_avail; struct em_buffer *tx_buffer; struct e1000_tx_desc *tx_desc, *eop_desc; - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; EM_TX_LOCK_ASSERT(adapter); @@ -3022,7 +3126,7 @@ lem_txeof(struct adapter *adapter) ++num_avail; if (tx_buffer->m_head) { - ifp->if_opackets++; + if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); bus_dmamap_sync(adapter->txtag, tx_buffer->map, BUS_DMASYNC_POSTWRITE); @@ -3057,13 +3161,29 @@ lem_txeof(struct adapter *adapter) adapter->next_tx_to_clean = first; adapter->num_tx_desc_avail = num_avail; +#ifdef NIC_SEND_COMBINING + if ((adapter->shadow_tdt & MIT_PENDING_TDT) == MIT_PENDING_TDT) { + /* a tdt write is pending, do it */ + E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), + 0xffff & adapter->shadow_tdt); + adapter->shadow_tdt = MIT_PENDING_INT; + } else { + adapter->shadow_tdt = 0; // disable + } +#endif /* NIC_SEND_COMBINING */ /* * If we have enough room, clear IFF_DRV_OACTIVE to * tell the stack that it is OK to send packets. * If there are no pending descriptors, clear the watchdog. */ if (adapter->num_tx_desc_avail > EM_TX_CLEANUP_THRESHOLD) { - ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; + if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); +#ifdef NIC_PARAVIRT + if (adapter->csb) { // XXX also csb_on ? + adapter->csb->guest_need_txkick = 2; /* acked */ + // XXX memory barrier + } +#endif /* NIC_PARAVIRT */ if (adapter->num_tx_desc_avail == adapter->num_tx_desc) { adapter->watchdog_check = FALSE; return; @@ -3184,8 +3304,7 @@ lem_allocate_receive_structures(struct adapter *adapter) } /* Create the spare map (used by getbuf) */ - error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT, - &adapter->rx_sparemap); + error = bus_dmamap_create(adapter->rxtag, 0, &adapter->rx_sparemap); if (error) { device_printf(dev, "%s: bus_dmamap_create failed: %d\n", __func__, error); @@ -3194,8 +3313,7 @@ lem_allocate_receive_structures(struct adapter *adapter) rx_buffer = adapter->rx_buffer_area; for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { - error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT, - &rx_buffer->map); + error = bus_dmamap_create(adapter->rxtag, 0, &rx_buffer->map); if (error) { device_printf(dev, "%s: bus_dmamap_create failed: %d\n", __func__, error); @@ -3222,7 +3340,7 @@ lem_setup_receive_structures(struct adapter *adapter) int i, error; #ifdef DEV_NETMAP /* we are already under lock */ - struct netmap_adapter *na = NA(adapter->ifp); + struct netmap_adapter *na = netmap_getna(adapter->ifp); struct netmap_slot *slot = netmap_reset(na, NR_RX, 0, 0); #endif @@ -3251,8 +3369,8 @@ lem_setup_receive_structures(struct adapter *adapter) uint64_t paddr; void *addr; - addr = PNMB(slot + si, &paddr); - netmap_load_map(adapter->rxtag, rx_buffer->map, addr); + addr = PNMB(na, slot + si, &paddr); + netmap_load_map(na, adapter->rxtag, rx_buffer->map, addr); /* Update descriptor */ adapter->rx_desc_base[i].buffer_addr = htole64(paddr); continue; @@ -3280,7 +3398,7 @@ lem_setup_receive_structures(struct adapter *adapter) static void lem_initialize_receive_unit(struct adapter *adapter) { - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; u64 bus_addr; u32 rctl, rxcsum; @@ -3345,14 +3463,14 @@ lem_initialize_receive_unit(struct adapter *adapter) break; } - if (ifp->if_mtu > ETHERMTU) + if (if_getmtu(ifp) > ETHERMTU) rctl |= E1000_RCTL_LPE; else rctl &= ~E1000_RCTL_LPE; /* Enable 82543 Receive Checksum Offload for TCP and UDP */ if ((adapter->hw.mac.type >= e1000_82543) && - (ifp->if_capenable & IFCAP_RXCSUM)) { + (if_getcapenable(ifp) & IFCAP_RXCSUM)) { rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM); rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL); E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum); @@ -3369,8 +3487,10 @@ lem_initialize_receive_unit(struct adapter *adapter) rctl = adapter->num_rx_desc - 1; /* default RDT value */ #ifdef DEV_NETMAP /* preserve buffers already made available to clients */ - if (ifp->if_capenable & IFCAP_NETMAP) - rctl -= nm_kr_rxspace(&NA(adapter->ifp)->rx_rings[0]); + if (if_getcapenable(ifp) & IFCAP_NETMAP) { + struct netmap_adapter *na = netmap_getna(adapter->ifp); + rctl -= nm_kr_rxspace(&na->rx_rings[0]); + } #endif /* DEV_NETMAP */ E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), rctl); @@ -3442,14 +3562,30 @@ lem_free_receive_structures(struct adapter *adapter) static bool lem_rxeof(struct adapter *adapter, int count, int *done) { - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; struct mbuf *mp; u8 status = 0, accept_frame = 0, eop = 0; u16 len, desc_len, prev_len_adj; int i, rx_sent = 0; struct e1000_rx_desc *current_desc; +#ifdef BATCH_DISPATCH + struct mbuf *mh = NULL, *mt = NULL; +#endif /* BATCH_DISPATCH */ +#ifdef NIC_PARAVIRT + int retries = 0; + struct paravirt_csb* csb = adapter->csb; + int csb_mode = csb && csb->guest_csb_on; + + //ND("clear guest_rxkick at %d", adapter->next_rx_desc_to_check); + if (csb_mode && csb->guest_need_rxkick) + csb->guest_need_rxkick = 0; +#endif /* NIC_PARAVIRT */ EM_RX_LOCK(adapter); + +#ifdef BATCH_DISPATCH + batch_again: +#endif /* BATCH_DISPATCH */ i = adapter->next_rx_desc_to_check; current_desc = &adapter->rx_desc_base[i]; bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, @@ -3462,19 +3598,45 @@ lem_rxeof(struct adapter *adapter, int count, int *done) } #endif /* DEV_NETMAP */ +#if 1 // XXX optimization ? if (!((current_desc->status) & E1000_RXD_STAT_DD)) { if (done != NULL) *done = rx_sent; EM_RX_UNLOCK(adapter); return (FALSE); } +#endif /* 0 */ - while (count != 0 && ifp->if_drv_flags & IFF_DRV_RUNNING) { + while (count != 0 && if_getdrvflags(ifp) & IFF_DRV_RUNNING) { struct mbuf *m = NULL; status = current_desc->status; - if ((status & E1000_RXD_STAT_DD) == 0) + if ((status & E1000_RXD_STAT_DD) == 0) { +#ifdef NIC_PARAVIRT + if (csb_mode) { + /* buffer not ready yet. Retry a few times before giving up */ + if (++retries <= adapter->rx_retries) { + continue; + } + if (csb->guest_need_rxkick == 0) { + // ND("set guest_rxkick at %d", adapter->next_rx_desc_to_check); + csb->guest_need_rxkick = 1; + // XXX memory barrier, status volatile ? + continue; /* double check */ + } + } + /* no buffer ready, give up */ +#endif /* NIC_PARAVIRT */ break; + } +#ifdef NIC_PARAVIRT + if (csb_mode) { + if (csb->guest_need_rxkick) + // ND("clear again guest_rxkick at %d", adapter->next_rx_desc_to_check); + csb->guest_need_rxkick = 0; + retries = 0; + } +#endif /* NIC_PARAVIRT */ mp = adapter->rx_buffer_area[i].m_head; /* @@ -3523,7 +3685,7 @@ lem_rxeof(struct adapter *adapter, int count, int *done) if (accept_frame) { if (lem_get_buf(adapter, i) != 0) { - ifp->if_iqdrops++; + if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); goto discard; } @@ -3553,8 +3715,8 @@ lem_rxeof(struct adapter *adapter, int count, int *done) } if (eop) { - adapter->fmp->m_pkthdr.rcvif = ifp; - ifp->if_ipackets++; + if_setrcvif(adapter->fmp, ifp); + if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); lem_receive_checksum(adapter, current_desc, adapter->fmp); #ifndef __NO_STRICT_ALIGNMENT @@ -3599,14 +3761,39 @@ discard: bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); +#ifdef NIC_PARAVIRT + if (csb_mode) { + /* the buffer at i has been already replaced by lem_get_buf() + * so it is safe to set guest_rdt = i and possibly send a kick. + * XXX see if we can optimize it later. + */ + csb->guest_rdt = i; + // XXX memory barrier + if (i == csb->host_rxkick_at) + E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), i); + } +#endif /* NIC_PARAVIRT */ /* Advance our pointers to the next descriptor. */ if (++i == adapter->num_rx_desc) i = 0; /* Call into the stack */ if (m != NULL) { +#ifdef BATCH_DISPATCH + if (adapter->batch_enable) { + if (mh == NULL) + mh = mt = m; + else + mt->m_nextpkt = m; + mt = m; + m->m_nextpkt = NULL; + rx_sent++; + current_desc = &adapter->rx_desc_base[i]; + continue; + } +#endif /* BATCH_DISPATCH */ adapter->next_rx_desc_to_check = i; EM_RX_UNLOCK(adapter); - (*ifp->if_input)(ifp, m); + if_input(ifp, m); EM_RX_LOCK(adapter); rx_sent++; i = adapter->next_rx_desc_to_check; @@ -3614,10 +3801,27 @@ discard: current_desc = &adapter->rx_desc_base[i]; } adapter->next_rx_desc_to_check = i; +#ifdef BATCH_DISPATCH + if (mh) { + EM_RX_UNLOCK(adapter); + while ( (mt = mh) != NULL) { + mh = mh->m_nextpkt; + mt->m_nextpkt = NULL; + if_input(ifp, mt); + } + EM_RX_LOCK(adapter); + i = adapter->next_rx_desc_to_check; /* in case of interrupts */ + if (count > 0) + goto batch_again; + } +#endif /* BATCH_DISPATCH */ /* Advance the E1000's Receive Queue #0 "Tail Pointer". */ if (--i < 0) i = adapter->num_rx_desc - 1; +#ifdef NIC_PARAVIRT + if (!csb_mode) /* filter out writes */ +#endif /* NIC_PARAVIRT */ E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), i); if (done != NULL) *done = rx_sent; @@ -3637,7 +3841,7 @@ discard: * copy ethernet header to the new mbuf. The new mbuf is prepended into the * existing mbuf chain. * - * Be aware, best performance of the 8254x is achived only when jumbo frame is + * Be aware, best performance of the 8254x is achieved only when jumbo frame is * not used at all on architectures with strict alignment. */ static int @@ -3719,12 +3923,12 @@ lem_receive_checksum(struct adapter *adapter, * config EVENT */ static void -lem_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) +lem_register_vlan(void *arg, if_t ifp, u16 vtag) { - struct adapter *adapter = ifp->if_softc; + struct adapter *adapter = if_getsoftc(ifp); u32 index, bit; - if (ifp->if_softc != arg) /* Not our event */ + if (if_getsoftc(ifp) != arg) /* Not our event */ return; if ((vtag == 0) || (vtag > 4095)) /* Invalid ID */ @@ -3736,7 +3940,7 @@ lem_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) adapter->shadow_vfta[index] |= (1 << bit); ++adapter->num_vlans; /* Re-init to load the changes */ - if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) + if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) lem_init_locked(adapter); EM_CORE_UNLOCK(adapter); } @@ -3746,12 +3950,12 @@ lem_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) * unconfig EVENT */ static void -lem_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) +lem_unregister_vlan(void *arg, if_t ifp, u16 vtag) { - struct adapter *adapter = ifp->if_softc; + struct adapter *adapter = if_getsoftc(ifp); u32 index, bit; - if (ifp->if_softc != arg) + if (if_getsoftc(ifp) != arg) return; if ((vtag == 0) || (vtag > 4095)) /* Invalid */ @@ -3763,7 +3967,7 @@ lem_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) adapter->shadow_vfta[index] &= ~(1 << bit); --adapter->num_vlans; /* Re-init to load the changes */ - if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) + if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) lem_init_locked(adapter); EM_CORE_UNLOCK(adapter); } @@ -3980,7 +4184,7 @@ static void lem_enable_wakeup(device_t dev) { struct adapter *adapter = device_get_softc(dev); - struct ifnet *ifp = adapter->ifp; + if_t ifp = adapter->ifp; u32 pmc, ctrl, ctrl_ext, rctl; u16 status; @@ -4005,10 +4209,10 @@ lem_enable_wakeup(device_t dev) ** Determine type of Wakeup: note that wol ** is set with all bits on by default. */ - if ((ifp->if_capenable & IFCAP_WOL_MAGIC) == 0) + if ((if_getcapenable(ifp) & IFCAP_WOL_MAGIC) == 0) adapter->wol &= ~E1000_WUFC_MAG; - if ((ifp->if_capenable & IFCAP_WOL_MCAST) == 0) + if ((if_getcapenable(ifp) & IFCAP_WOL_MCAST) == 0) adapter->wol &= ~E1000_WUFC_MC; else { rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); @@ -4028,7 +4232,7 @@ lem_enable_wakeup(device_t dev) /* Request PME */ status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2); status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE); - if (ifp->if_capenable & IFCAP_WOL) + if (if_getcapenable(ifp) & IFCAP_WOL) status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2); @@ -4197,7 +4401,6 @@ lem_fill_descriptors (bus_addr_t address, u32 length, static void lem_update_stats_counters(struct adapter *adapter) { - struct ifnet *ifp; if(adapter->hw.phy.media_type == e1000_media_type_copper || (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) { @@ -4272,19 +4475,29 @@ lem_update_stats_counters(struct adapter *adapter) adapter->stats.tsctfc += E1000_READ_REG(&adapter->hw, E1000_TSCTFC); } - ifp = adapter->ifp; - - ifp->if_collisions = adapter->stats.colc; +} - /* Rx Errors */ - ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc + - adapter->stats.crcerrs + adapter->stats.algnerrc + - adapter->stats.ruc + adapter->stats.roc + - adapter->stats.mpc + adapter->stats.cexterr; +static uint64_t +lem_get_counter(if_t ifp, ift_counter cnt) +{ + struct adapter *adapter; - /* Tx Errors */ - ifp->if_oerrors = adapter->stats.ecol + - adapter->stats.latecol + adapter->watchdog_events; + adapter = if_getsoftc(ifp); + + switch (cnt) { + case IFCOUNTER_COLLISIONS: + return (adapter->stats.colc); + case IFCOUNTER_IERRORS: + return (adapter->dropped_pkts + adapter->stats.rxerrc + + adapter->stats.crcerrs + adapter->stats.algnerrc + + adapter->stats.ruc + adapter->stats.roc + + adapter->stats.mpc + adapter->stats.cexterr); + case IFCOUNTER_OERRORS: + return (adapter->stats.ecol + adapter->stats.latecol + + adapter->watchdog_events); + default: + return (if_get_counter_default(ifp, cnt)); + } } /* Export a single 32-bit register via a read-only sysctl. */ @@ -4316,12 +4529,12 @@ lem_add_hw_stats(struct adapter *adapter) struct sysctl_oid_list *stat_list; /* Driver Statistics */ - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_alloc_fail", - CTLFLAG_RD, &adapter->mbuf_alloc_failed, - "Std mbuf failed"); SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "cluster_alloc_fail", CTLFLAG_RD, &adapter->mbuf_cluster_failed, "Std mbuf cluster failed"); + SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_fail", + CTLFLAG_RD, &adapter->mbuf_defrag_failed, + "Defragmenting mbuf chain failed"); SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", CTLFLAG_RD, &adapter->dropped_pkts, "Driver dropped packets"); diff --git a/freebsd/sys/dev/e1000/if_lem.h b/freebsd/sys/dev/e1000/if_lem.h index 235277d7..4a27c34b 100644 --- a/freebsd/sys/dev/e1000/if_lem.h +++ b/freebsd/sys/dev/e1000/if_lem.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2011, Intel Corporation + Copyright (c) 2001-2015, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -236,10 +236,8 @@ #define HW_DEBUGOUT1(S, A) if (DEBUG_HW) printf(S "\n", A) #define HW_DEBUGOUT2(S, A, B) if (DEBUG_HW) printf(S "\n", A, B) -#define EM_MAX_SCATTER 64 +#define EM_MAX_SCATTER 40 #define EM_VFTA_SIZE 128 -#define EM_TSO_SIZE (65535 + sizeof(struct ether_vlan_header)) -#define EM_TSO_SEG_SIZE 4096 /* Max dma segment size */ #define EM_MSIX_MASK 0x01F00000 /* For 82574 use */ #define ETH_ZLEN 60 #define ETH_ADDR_LEN 6 @@ -265,6 +263,13 @@ #define PICOSECS_PER_TICK 20833 #define TSYNC_PORT 319 /* UDP port for the protocol */ +#ifdef NIC_PARAVIRT +#define E1000_PARA_SUBDEV 0x1101 /* special id */ +#define E1000_CSBAL 0x02830 /* csb phys. addr. low */ +#define E1000_CSBAH 0x02834 /* csb phys. addr. hi */ +#include <net/paravirt.h> +#endif /* NIC_PARAVIRT */ + /* * Bus dma allocation structure used by * e1000_dma_malloc and e1000_dma_free. @@ -288,15 +293,12 @@ struct em_int_delay_info { /* Our adapter structure */ struct adapter { - struct ifnet *ifp; -#if __FreeBSD_version >= 800000 - struct buf_ring *br; -#endif + if_t ifp; struct e1000_hw hw; /* FreeBSD operating-system-specific structures. */ struct e1000_osdep osdep; - struct device *dev; + device_t dev; struct cdev *led_dev; struct resource *memory; @@ -413,17 +415,17 @@ struct adapter { /* Misc stats maintained by the driver */ unsigned long dropped_pkts; - unsigned long mbuf_alloc_failed; + unsigned long link_irq; unsigned long mbuf_cluster_failed; + unsigned long mbuf_defrag_failed; unsigned long no_tx_desc_avail1; unsigned long no_tx_desc_avail2; + unsigned long no_tx_dma_setup; unsigned long no_tx_map_avail; - unsigned long no_tx_dma_setup; unsigned long watchdog_events; - unsigned long rx_overruns; unsigned long rx_irq; + unsigned long rx_overruns; unsigned long tx_irq; - unsigned long link_irq; /* 82547 workaround */ uint32_t tx_fifo_size; @@ -437,6 +439,26 @@ struct adapter { boolean_t pcix_82544; boolean_t in_detach; +#ifdef NIC_SEND_COMBINING + /* 0 = idle; 1xxxx int-pending; 3xxxx int + d pending + tdt */ +#define MIT_PENDING_INT 0x10000 /* pending interrupt */ +#define MIT_PENDING_TDT 0x30000 /* both intr and tdt write are pending */ + uint32_t shadow_tdt; + uint32_t sc_enable; +#endif /* NIC_SEND_COMBINING */ +#ifdef BATCH_DISPATCH + uint32_t batch_enable; +#endif /* BATCH_DISPATCH */ + +#ifdef NIC_PARAVIRT + struct em_dma_alloc csb_mem; /* phys address */ + struct paravirt_csb *csb; /* virtual addr */ + uint32_t rx_retries; /* optimize rx loop */ + uint32_t tdt_csb_count;// XXX stat + uint32_t tdt_reg_count;// XXX stat + uint32_t tdt_int_count;// XXX stat + uint32_t guest_need_kick_count;// XXX stat +#endif /* NIC_PARAVIRT */ struct e1000_hw_stats stats; }; |