Update to FreeBSD head 2017-08-01

Git mirror commit f5002f5e5f78cae9f0269d812dc0aedb0339312c.

Update #3472.

50 files changed, 3077 insertions, 665 deletions

diff --git a/freebsd/sys/dev/e1000/if_em.c b/freebsd/sys/dev/e1000/if_em.c
index 69381438..f5a0b94e 100644
--- a/freebsd/sys/dev/e1000/if_em.c
+++ b/freebsd/sys/dev/e1000/if_em.c
@@ -205,7 +205,7 @@ static pci_vendor_info_t igb_vendor_info_array[] =
 	PVID(0x8086, E1000_DEV_ID_I210_COPPER_OEM1, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_I210_COPPER_FLASHLESS, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_I210_SERDES_FLASHLESS, "Intel(R) PRO/1000 PCI-Express Network Driver"),
-	PVID(0x8086, E1000_DEV_ID_I210_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"), 
+	PVID(0x8086, E1000_DEV_ID_I210_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_I210_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_I210_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_I211_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
@@ -233,8 +233,8 @@ static int	em_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs
 static void	em_if_queues_free(if_ctx_t ctx);
 
 static uint64_t	em_if_get_counter(if_ctx_t, ift_counter);
-static void	em_if_init(if_ctx_t ctx); 
-static void	em_if_stop(if_ctx_t ctx); 
+static void	em_if_init(if_ctx_t ctx);
+static void	em_if_stop(if_ctx_t ctx);
 static void	em_if_media_status(if_ctx_t, struct ifmediareq *);
 static int	em_if_media_change(if_ctx_t ctx);
 static int	em_if_mtu_set(if_ctx_t ctx, uint32_t mtu);
@@ -359,11 +359,11 @@ static device_method_t em_if_methods[] = {
 	DEVMETHOD(ifdi_detach, em_if_detach),
 	DEVMETHOD(ifdi_shutdown, em_if_shutdown),
 	DEVMETHOD(ifdi_suspend, em_if_suspend),
-	DEVMETHOD(ifdi_resume, em_if_resume), 
+	DEVMETHOD(ifdi_resume, em_if_resume),
 	DEVMETHOD(ifdi_init, em_if_init),
 	DEVMETHOD(ifdi_stop, em_if_stop),
 	DEVMETHOD(ifdi_msix_intr_assign, em_if_msix_intr_assign),
-	DEVMETHOD(ifdi_intr_enable, em_if_enable_intr), 
+	DEVMETHOD(ifdi_intr_enable, em_if_enable_intr),
 	DEVMETHOD(ifdi_intr_disable, em_if_disable_intr),
 	DEVMETHOD(ifdi_tx_queues_alloc, em_if_tx_queues_alloc),
 	DEVMETHOD(ifdi_rx_queues_alloc, em_if_rx_queues_alloc),
@@ -1027,7 +1027,7 @@ em_if_attach_post(if_ctx_t ctx)
 	/* Non-AMT based hardware can now take control from firmware */
 	if (adapter->has_manage && !adapter->has_amt)
 		em_get_hw_control(adapter);
-	
+
 	INIT_DEBUGOUT("em_if_attach_post: end");
 
 	return (error);
@@ -1403,7 +1403,7 @@ em_msix_link(void *arg)
 	u32 reg_icr;
 
 	++adapter->link_irq;
-	MPASS(adapter->hw.back != NULL); 
+	MPASS(adapter->hw.back != NULL);
 	reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
 
 	if (reg_icr & E1000_ICR_RXO)
@@ -1665,9 +1665,7 @@ em_if_timer(if_ctx_t ctx, uint16_t qid)
 	if (qid != 0)
 		return;
 
-	em_if_update_admin_status(ctx);
-	em_update_stats_counters(adapter);
-
+	iflib_admin_intr_deferred(ctx);
 	/* Reset LAA into RAR[0] on 82571 */
 	if ((adapter->hw.mac.type == e1000_82571) &&
 	    e1000_get_laa_state_82571(&adapter->hw))
@@ -1693,8 +1691,9 @@ em_if_update_admin_status(if_ctx_t ctx)
 	struct e1000_hw *hw = &adapter->hw;
 	struct ifnet *ifp = iflib_get_ifp(ctx);
 	device_t dev = iflib_get_dev(ctx);
-	u32 link_check = 0;
+	u32 link_check, thstat, ctrl;
 
+	link_check = thstat = ctrl = 0;
 	/* Get the cached link value or read phy for real */
 	switch (hw->phy.media_type) {
 	case e1000_media_type_copper:
@@ -1719,11 +1718,21 @@ em_if_update_admin_status(if_ctx_t ctx)
 		e1000_check_for_link(hw);
 		link_check = adapter->hw.mac.serdes_has_link;
 		break;
-	default:
+	/* VF device is type_unknown */
 	case e1000_media_type_unknown:
+		e1000_check_for_link(hw);
+		link_check = !hw->mac.get_link_status;
+		/* FALLTHROUGH */
+	default:
 		break;
 	}
 
+	/* Check for thermal downshift or shutdown */
+	if (hw->mac.type == e1000_i350) {
+		thstat = E1000_READ_REG(hw, E1000_THSTAT);
+		ctrl = E1000_READ_REG(hw, E1000_CTRL_EXT);
+	}
+
 	/* Now check for a transition */
 	if (link_check && (adapter->link_active == 0)) {
 		e1000_get_speed_and_duplex(hw, &adapter->link_speed,
@@ -1745,6 +1754,21 @@ em_if_update_admin_status(if_ctx_t ctx)
 		adapter->link_active = 1;
 		adapter->smartspeed = 0;
 		if_setbaudrate(ifp, adapter->link_speed * 1000000);
+		if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) &&
+		    (thstat & E1000_THSTAT_LINK_THROTTLE))
+			device_printf(dev, "Link: thermal downshift\n");
+		/* Delay Link Up for Phy update */
+		if (((hw->mac.type == e1000_i210) ||
+		    (hw->mac.type == e1000_i211)) &&
+		    (hw->phy.id == I210_I_PHY_ID))
+			msec_delay(I210_LINK_DELAY);
+		/* Reset if the media type changed. */
+		if ((hw->dev_spec._82575.media_changed) &&
+			(adapter->hw.mac.type >= igb_mac_min)) {
+			hw->dev_spec._82575.media_changed = false;
+			adapter->flags |= IGB_MEDIA_RESET;
+			em_reset(ctx);
+		}
 		iflib_link_state_change(ctx, LINK_STATE_UP, ifp->if_baudrate);
 		printf("Link state changed to up\n");
 	} else if (!link_check && (adapter->link_active == 1)) {
@@ -1757,6 +1781,7 @@ em_if_update_admin_status(if_ctx_t ctx)
 		iflib_link_state_change(ctx, LINK_STATE_DOWN, ifp->if_baudrate);
 		printf("link state changed to down\n");
 	}
+	em_update_stats_counters(adapter);
 
 	E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_LINK | E1000_IMS_LSC);
 }
@@ -2212,6 +2237,114 @@ lem_smartspeed(struct adapter *adapter)
 		adapter->smartspeed = 0;
 }
 
+/*********************************************************************
+ *
+ *  Initialize the DMA Coalescing feature
+ *
+ **********************************************************************/
+static void
+igb_init_dmac(struct adapter *adapter, u32 pba)
+{
+	device_t	dev = adapter->dev;
+	struct e1000_hw *hw = &adapter->hw;
+	u32 		dmac, reg = ~E1000_DMACR_DMAC_EN;
+	u16		hwm;
+	u16		max_frame_size;
+
+	if (hw->mac.type == e1000_i211)
+		return;
+
+	max_frame_size = adapter->shared->isc_max_frame_size;
+	if (hw->mac.type > e1000_82580) {
+
+		if (adapter->dmac == 0) { /* Disabling it */
+			E1000_WRITE_REG(hw, E1000_DMACR, reg);
+			return;
+		} else
+			device_printf(dev, "DMA Coalescing enabled\n");
+
+		/* Set starting threshold */
+		E1000_WRITE_REG(hw, E1000_DMCTXTH, 0);
+
+		hwm = 64 * pba - max_frame_size / 16;
+		if (hwm < 64 * (pba - 6))
+			hwm = 64 * (pba - 6);
+		reg = E1000_READ_REG(hw, E1000_FCRTC);
+		reg &= ~E1000_FCRTC_RTH_COAL_MASK;
+		reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
+		    & E1000_FCRTC_RTH_COAL_MASK);
+		E1000_WRITE_REG(hw, E1000_FCRTC, reg);
+
+
+		dmac = pba - max_frame_size / 512;
+		if (dmac < pba - 10)
+			dmac = pba - 10;
+		reg = E1000_READ_REG(hw, E1000_DMACR);
+		reg &= ~E1000_DMACR_DMACTHR_MASK;
+		reg = ((dmac << E1000_DMACR_DMACTHR_SHIFT)
+		    & E1000_DMACR_DMACTHR_MASK);
+
+		/* transition to L0x or L1 if available..*/
+		reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
+
+		/* Check if status is 2.5Gb backplane connection
+		* before configuration of watchdog timer, which is
+		* in msec values in 12.8usec intervals
+		* watchdog timer= msec values in 32usec intervals
+		* for non 2.5Gb connection
+		*/
+		if (hw->mac.type == e1000_i354) {
+			int status = E1000_READ_REG(hw, E1000_STATUS);
+			if ((status & E1000_STATUS_2P5_SKU) &&
+			    (!(status & E1000_STATUS_2P5_SKU_OVER)))
+				reg |= ((adapter->dmac * 5) >> 6);
+			else
+				reg |= (adapter->dmac >> 5);
+		} else {
+			reg |= (adapter->dmac >> 5);
+		}
+
+		E1000_WRITE_REG(hw, E1000_DMACR, reg);
+
+		E1000_WRITE_REG(hw, E1000_DMCRTRH, 0);
+
+		/* Set the interval before transition */
+		reg = E1000_READ_REG(hw, E1000_DMCTLX);
+		if (hw->mac.type == e1000_i350)
+			reg |= IGB_DMCTLX_DCFLUSH_DIS;
+		/*
+		** in 2.5Gb connection, TTLX unit is 0.4 usec
+		** which is 0x4*2 = 0xA. But delay is still 4 usec
+		*/
+		if (hw->mac.type == e1000_i354) {
+			int status = E1000_READ_REG(hw, E1000_STATUS);
+			if ((status & E1000_STATUS_2P5_SKU) &&
+			    (!(status & E1000_STATUS_2P5_SKU_OVER)))
+				reg |= 0xA;
+			else
+				reg |= 0x4;
+		} else {
+			reg |= 0x4;
+		}
+
+		E1000_WRITE_REG(hw, E1000_DMCTLX, reg);
+
+		/* free space in tx packet buffer to wake from DMA coal */
+		E1000_WRITE_REG(hw, E1000_DMCTXTH, (IGB_TXPBSIZE -
+		    (2 * max_frame_size)) >> 6);
+
+		/* make low power state decision controlled by DMA coal */
+		reg = E1000_READ_REG(hw, E1000_PCIEMISC);
+		reg &= ~E1000_PCIEMISC_LX_DECISION;
+		E1000_WRITE_REG(hw, E1000_PCIEMISC, reg);
+
+	} else if (hw->mac.type == e1000_82580) {
+		u32 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
+		E1000_WRITE_REG(hw, E1000_PCIEMISC,
+		    reg & ~E1000_PCIEMISC_LX_DECISION);
+		E1000_WRITE_REG(hw, E1000_DMACR, 0);
+	}
+}
 
 static void
 em_reset(if_ctx_t ctx)
@@ -2224,6 +2357,8 @@ em_reset(if_ctx_t ctx)
 	u32 pba;
 
 	INIT_DEBUGOUT("em_reset: begin");
+	/* Let the firmware know the OS is in control */
+	em_get_hw_control(adapter);
 
 	/* Set up smart power down as default off on newer adapters. */
 	if (!em_smart_pwr_down && (hw->mac.type == e1000_82571 ||
@@ -2402,15 +2537,15 @@ em_reset(if_ctx_t ctx)
 	case e1000_vfadapt_i350:
 		/* 16-byte granularity */
 		hw->fc.low_water = hw->fc.high_water - 16;
-		break; 
-        case e1000_ich9lan:
-        case e1000_ich10lan:
+		break;
+	case e1000_ich9lan:
+	case e1000_ich10lan:
 		if (if_getmtu(ifp) > ETHERMTU) {
 			hw->fc.high_water = 0x2800;
 			hw->fc.low_water = hw->fc.high_water - 8;
 			break;
 		}
-		/* else fall thru */
+		/* FALLTHROUGH */
 	default:
 		if (hw->mac.type == e1000_80003es2lan)
 			hw->fc.pause_time = 0xFFFF;
@@ -2419,13 +2554,24 @@ em_reset(if_ctx_t ctx)
 
 	/* Issue a global reset */
 	e1000_reset_hw(hw);
-	E1000_WRITE_REG(hw, E1000_WUFC, 0);
-	em_disable_aspm(adapter);
+	if (adapter->hw.mac.type >= igb_mac_min) {
+		E1000_WRITE_REG(hw, E1000_WUC, 0);
+	} else {
+		E1000_WRITE_REG(hw, E1000_WUFC, 0);
+		em_disable_aspm(adapter);
+	}
+	if (adapter->flags & IGB_MEDIA_RESET) {
+		e1000_setup_init_funcs(hw, TRUE);
+		e1000_get_bus_info(hw);
+		adapter->flags &= ~IGB_MEDIA_RESET;
+	}
 	/* and a re-init */
 	if (e1000_init_hw(hw) < 0) {
 		device_printf(dev, "Hardware Initialization Failed\n");
 		return;
 	}
+	if (adapter->hw.mac.type >= igb_mac_min)
+		igb_init_dmac(adapter, pba);
 
 	E1000_WRITE_REG(hw, E1000_VET, ETHERTYPE_VLAN);
 	e1000_get_phy_info(hw);
@@ -2466,7 +2612,7 @@ em_initialize_rss_mapping(struct adapter *adapter)
 	for (i = 0; i < 32; ++i)
 		E1000_WRITE_REG(hw, E1000_RETA(i), reta);
 
-	E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_RSS_ENABLE_2Q | 
+	E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_RSS_ENABLE_2Q |
 			E1000_MRQC_RSS_FIELD_IPV4_TCP |
 			E1000_MRQC_RSS_FIELD_IPV4 |
 			E1000_MRQC_RSS_FIELD_IPV6_TCP_EX |
@@ -2553,8 +2699,7 @@ igb_initialize_rss_mapping(struct adapter *adapter)
 	arc4rand(&rss_key, sizeof(rss_key), 0);
 #endif
 	for (i = 0; i < 10; i++)
-		E1000_WRITE_REG_ARRAY(hw,
-		    E1000_RSSRK(0), i, rss_key[i]);
+		E1000_WRITE_REG_ARRAY(hw, E1000_RSSRK(0), i, rss_key[i]);
 
 	/*
 	 * Configure the RSS fields to hash upon.
@@ -2621,7 +2766,7 @@ em_setup_interface(if_ctx_t ctx)
 	/* Enable only WOL MAGIC by default */
 	if (adapter->wol) {
 		if_setcapenablebit(ifp, IFCAP_WOL_MAGIC,
-			     IFCAP_WOL_MCAST| IFCAP_WOL_UCAST);
+			    IFCAP_WOL_MCAST| IFCAP_WOL_UCAST);
 	} else {
 		if_setcapenablebit(ifp, 0, IFCAP_WOL_MAGIC |
 			     IFCAP_WOL_MCAST| IFCAP_WOL_UCAST);
@@ -2693,7 +2838,7 @@ em_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs
 		txr->tx_base = (struct e1000_tx_desc *)vaddrs[i*ntxqs];
 		txr->tx_paddr = paddrs[i*ntxqs];
 	}
-	
+
 	device_printf(iflib_get_dev(ctx), "allocated for %d tx_queues\n", adapter->tx_num_queues);
 	return (0);
 fail:
@@ -2718,7 +2863,7 @@ em_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nrxqs
 	    adapter->rx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 		device_printf(iflib_get_dev(ctx), "Unable to allocate queue memory\n");
 		error = ENOMEM;
-		goto fail; 
+		goto fail;
 	}
 
 	for (i = 0, que = adapter->rx_queues; i < nrxqsets; i++, que++) {
@@ -2758,7 +2903,7 @@ em_if_queues_free(if_ctx_t ctx)
 			txr->tx_rsq = NULL;
 		}
 		free(adapter->tx_queues, M_DEVBUF);
-		adapter->tx_queues = NULL; 
+		adapter->tx_queues = NULL;
 	}
 
 	if (rx_que != NULL) {
@@ -3033,7 +3178,7 @@ em_initialize_receive_unit(if_ctx_t ctx)
 		u64 bus_addr = rxr->rx_paddr;
 #if 0
 		u32 rdt = adapter->rx_num_queues -1;  /* default */
-#endif		
+#endif
 
 		E1000_WRITE_REG(hw, E1000_RDLEN(i),
 		    scctx->isc_nrxd[0] * sizeof(union e1000_rx_desc_extended));
@@ -3088,7 +3233,7 @@ em_initialize_receive_unit(if_ctx_t ctx)
 			srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
 			rctl |= E1000_RCTL_SZ_2048;
 		}
-	
+
 		/*
 		 * If TX flow control is disabled and there's >1 queue defined,
 		 * enable DROP.
@@ -3126,7 +3271,7 @@ em_initialize_receive_unit(if_ctx_t ctx)
 			rxdctl &= 0xFFF00000;
 			rxdctl |= IGB_RX_PTHRESH;
 			rxdctl |= IGB_RX_HTHRESH << 8;
-			rxdctl |= IGB_RX_WTHRESH << 16; 
+			rxdctl |= IGB_RX_WTHRESH << 16;
 			E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl);
 		}		
 	} else if (adapter->hw.mac.type >= e1000_pch2lan) {
@@ -3255,7 +3400,7 @@ em_if_disable_intr(if_ctx_t ctx)
 /*
  * Bit of a misnomer, what this really means is
  * to enable OS management of the system... aka
- * to disable special hardware management features 
+ * to disable special hardware management features
  */
 static void
 em_init_manageability(struct adapter *adapter)
@@ -3309,6 +3454,9 @@ em_get_hw_control(struct adapter *adapter)
 {
 	u32 ctrl_ext, swsm;
 
+	if (adapter->vf_ifp)
+		return;
+
 	if (adapter->hw.mac.type == e1000_82573) {
 		swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
 		E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
@@ -3319,7 +3467,6 @@ em_get_hw_control(struct adapter *adapter)
 	ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
 	E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
 	    ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
-	return;
 }
 
 /*
@@ -3639,7 +3786,7 @@ static void
 em_if_led_func(if_ctx_t ctx, int onoff)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
- 
+
 	if (onoff) {
 		e1000_setup_led(&adapter->hw);
 		e1000_led_on(&adapter->hw);
@@ -3785,7 +3932,7 @@ static uint64_t
 em_if_get_counter(if_ctx_t ctx, ift_counter cnt)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
-	struct ifnet *ifp = iflib_get_ifp(ctx); 
+	struct ifnet *ifp = iflib_get_ifp(ctx);
 
 	switch (cnt) {
 	case IFCOUNTER_COLLISIONS:
@@ -3822,7 +3969,7 @@ static void
 em_add_hw_stats(struct adapter *adapter)
 {
 	device_t dev = iflib_get_dev(adapter->ctx);
-        struct em_tx_queue *tx_que = adapter->tx_queues;
+	struct em_tx_queue *tx_que = adapter->tx_queues;
 	struct em_rx_queue *rx_que = adapter->rx_queues;
 
 	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
@@ -4067,7 +4214,7 @@ em_add_hw_stats(struct adapter *adapter)
 
 	/* Interrupt Stats */
 
-	int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts", 
+	int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts",
 				    CTLFLAG_RD, NULL, "Interrupt Statistics");
 	int_list = SYSCTL_CHILDREN(int_node);
 
@@ -4244,7 +4391,7 @@ em_set_flowcntl(SYSCTL_HANDLER_ARGS)
 	case e1000_fc_none:
 		adapter->hw.fc.requested_mode = input;
 		adapter->fc = input;
-                break;
+		break;
 	default:
 		/* Do nothing */
 		return (error);
@@ -4292,7 +4439,7 @@ em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
 	if (result == 1) {
 		adapter = (struct adapter *) arg1;
 		em_print_debug_info(adapter);
-        }
+	}
 
 	return (error);
 }
diff --git a/freebsd/sys/dev/e1000/if_em.h b/freebsd/sys/dev/e1000/if_em.h
index 79af551c..67d97e17 100644
--- a/freebsd/sys/dev/e1000/if_em.h
+++ b/freebsd/sys/dev/e1000/if_em.h
@@ -235,6 +235,27 @@
 #define EM_EEPROM_APME			0x400;
 #define EM_82544_APME			0x0004;
 
+
+/* Support AutoMediaDetect for Marvell M88 PHY in i354 */
+#define IGB_MEDIA_RESET			(1 << 0)
+
+/* Define the starting Interrupt rate per Queue */
+#define IGB_INTS_PER_SEC        8000
+#define IGB_DEFAULT_ITR         ((1000000/IGB_INTS_PER_SEC) << 2)
+
+#define IGB_LINK_ITR            2000
+#define I210_LINK_DELAY		1000
+
+#define IGB_MAX_SCATTER		40
+#define IGB_VFTA_SIZE		128
+#define IGB_BR_SIZE		4096	/* ring buf size */
+#define IGB_TSO_SIZE		(65535 + sizeof(struct ether_vlan_header))
+#define IGB_TSO_SEG_SIZE	4096	/* Max dma segment size */
+#define IGB_TXPBSIZE		20408
+#define IGB_HDR_BUF		128
+#define IGB_PKTTYPE_MASK	0x0000FFF0
+#define IGB_DMCTLX_DCFLUSH_DIS	0x80000000  /* Disable DMA Coalesce Flush */
+
 /*
  * Driver state logic for the detection of a hung state
  * in hardware.  Set TX_HUNG whenever a TX packet is used
@@ -455,11 +476,11 @@ struct adapter {
 	struct ifmedia	*media;
 	int		msix;
 	int		if_flags;
-	int		min_frame_size;
 	int		em_insert_vlan_header;
 	u32		ims;
 	bool		in_detach;
 
+	u32		flags;
 	/* Task for FAST handling */
 	struct grouptask link_task;
 
@@ -514,6 +535,7 @@ struct adapter {
 	unsigned long	watchdog_events;
 
 	struct e1000_hw_stats stats;
+	u16		vf_ifp;
 };
 
 /********************************************************************************
diff --git a/freebsd/sys/dev/evdev/cdev.c b/freebsd/sys/dev/evdev/cdev.c
index b08a2440..10f4e77e 100644
--- a/freebsd/sys/dev/evdev/cdev.c
+++ b/freebsd/sys/dev/evdev/cdev.c
@@ -2,7 +2,7 @@
 
 /*-
  * Copyright (c) 2014 Jakub Wojciech Klama <jceel@FreeBSD.org>
- * Copyright (c) 2015-2016 Vladimir Kondratyev <wulf@cicgroup.ru>
+ * Copyright (c) 2015-2016 Vladimir Kondratyev <wulf@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,24 +31,23 @@
 
 #include <rtems/bsd/local/opt_evdev.h>
 
-#include <sys/types.h>
-#include <sys/bitstring.h>
-#include <sys/systm.h>
 #include <sys/param.h>
-#include <sys/kernel.h>
+#include <sys/bitstring.h>
 #include <sys/conf.h>
-#include <sys/uio.h>
-#include <sys/proc.h>
-#include <sys/poll.h>
 #include <sys/filio.h>
 #include <sys/fcntl.h>
-#include <sys/selinfo.h>
+#include <sys/kernel.h>
 #include <sys/malloc.h>
+#include <sys/poll.h>
+#include <sys/proc.h>
+#include <sys/selinfo.h>
+#include <sys/systm.h>
 #include <sys/time.h>
+#include <sys/uio.h>
 
-#include <dev/evdev/input.h>
 #include <dev/evdev/evdev.h>
 #include <dev/evdev/evdev_private.h>
+#include <dev/evdev/input.h>
 
 #ifdef EVDEV_DEBUG
 #define	debugf(client, fmt, args...)	printf("evdev cdev: "fmt"\n", ##args)
diff --git a/freebsd/sys/dev/evdev/evdev.c b/freebsd/sys/dev/evdev/evdev.c
index 185e48f7..b3c786a5 100644
--- a/freebsd/sys/dev/evdev/evdev.c
+++ b/freebsd/sys/dev/evdev/evdev.c
@@ -2,7 +2,7 @@
 
 /*-
  * Copyright (c) 2014 Jakub Wojciech Klama <jceel@FreeBSD.org>
- * Copyright (c) 2015-2016 Vladimir Kondratyev <wulf@cicgroup.ru>
+ * Copyright (c) 2015-2016 Vladimir Kondratyev <wulf@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,19 +31,18 @@
 
 #include <rtems/bsd/local/opt_evdev.h>
 
-#include <sys/types.h>
-#include <sys/systm.h>
 #include <sys/param.h>
-#include <sys/kernel.h>
-#include <sys/module.h>
+#include <sys/bitstring.h>
 #include <sys/conf.h>
+#include <sys/kernel.h>
 #include <sys/malloc.h>
-#include <sys/bitstring.h>
+#include <sys/module.h>
 #include <sys/sysctl.h>
+#include <sys/systm.h>
 
-#include <dev/evdev/input.h>
 #include <dev/evdev/evdev.h>
 #include <dev/evdev/evdev_private.h>
+#include <dev/evdev/input.h>
 
 #ifdef EVDEV_DEBUG
 #define	debugf(evdev, fmt, args...)	printf("evdev: " fmt "\n", ##args)
@@ -764,14 +763,11 @@ evdev_push_event(struct evdev_dev *evdev, uint16_t type, uint16_t code,
     int32_t value)
 {
 
-	if (evdev->ev_lock_type != EV_LOCK_INTERNAL)
-		EVDEV_LOCK_ASSERT(evdev);
-
 	if (evdev_check_event(evdev, type, code, value) != 0)
 		return (EINVAL);
 
-	if (evdev->ev_lock_type == EV_LOCK_INTERNAL)
-		EVDEV_LOCK(evdev);
+	EVDEV_ENTER(evdev);
+
 	evdev_modify_event(evdev, type, code, &value);
 	if (type == EV_SYN && code == SYN_REPORT &&
 	     bit_test(evdev->ev_flags, EVDEV_FLAG_MT_AUTOREL))
@@ -780,8 +776,8 @@ evdev_push_event(struct evdev_dev *evdev, uint16_t type, uint16_t code,
 	    bit_test(evdev->ev_flags, EVDEV_FLAG_MT_STCOMPAT))
 		evdev_send_mt_compat(evdev);
 	evdev_send_event(evdev, type, code, value);
-	if (evdev->ev_lock_type == EV_LOCK_INTERNAL)
-		EVDEV_UNLOCK(evdev);
+
+	EVDEV_EXIT(evdev);
 
 	return (0);
 }
diff --git a/freebsd/sys/dev/evdev/evdev_mt.c b/freebsd/sys/dev/evdev/evdev_mt.c
index 6412544b..d0eb48f4 100644
--- a/freebsd/sys/dev/evdev/evdev_mt.c
+++ b/freebsd/sys/dev/evdev/evdev_mt.c
@@ -1,7 +1,7 @@
 #include <machine/rtems-bsd-kernel-space.h>
 
 /*-
- * Copyright (c) 2016 Vladimir Kondratyev <wulf@cicgroup.ru>
+ * Copyright (c) 2016 Vladimir Kondratyev <wulf@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -29,14 +29,14 @@
  */
 
 #include <sys/param.h>
-#include <sys/malloc.h>
 #include <sys/lock.h>
+#include <sys/malloc.h>
 #include <sys/mutex.h>
 #include <sys/systm.h>
 
-#include <dev/evdev/input.h>
 #include <dev/evdev/evdev.h>
 #include <dev/evdev/evdev_private.h>
+#include <dev/evdev/input.h>
 
 #ifdef DEBUG
 #define	debugf(fmt, args...)	printf("evdev: " fmt "\n", ##args)
@@ -226,13 +226,9 @@ void
 evdev_push_nfingers(struct evdev_dev *evdev, int32_t nfingers)
 {
 
-	if (evdev->ev_lock_type == EV_LOCK_INTERNAL)
-		EVDEV_LOCK(evdev);
-	else
-		EVDEV_LOCK_ASSERT(evdev);
+	EVDEV_ENTER(evdev);
 	evdev_send_nfingers(evdev, nfingers);
-	if (evdev->ev_lock_type == EV_LOCK_INTERNAL)
-		EVDEV_UNLOCK(evdev);
+	EVDEV_EXIT(evdev);
 }
 
 void
@@ -266,13 +262,9 @@ void
 evdev_push_mt_compat(struct evdev_dev *evdev)
 {
 
-	if (evdev->ev_lock_type == EV_LOCK_INTERNAL)
-		EVDEV_LOCK(evdev);
-	else
-		EVDEV_LOCK_ASSERT(evdev);
+	EVDEV_ENTER(evdev);
 	evdev_send_mt_compat(evdev);
-	if (evdev->ev_lock_type == EV_LOCK_INTERNAL)
-		EVDEV_UNLOCK(evdev);
+	EVDEV_EXIT(evdev);
 }
 
 void
diff --git a/freebsd/sys/dev/evdev/evdev_private.h b/freebsd/sys/dev/evdev/evdev_private.h
index b3de1bf0..05206a9d 100644
--- a/freebsd/sys/dev/evdev/evdev_private.h
+++ b/freebsd/sys/dev/evdev/evdev_private.h
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2014 Jakub Wojciech Klama <jceel@FreeBSD.org>
- * Copyright (c) 2015-2016 Vladimir Kondratyev <wulf@cicgroup.ru>
+ * Copyright (c) 2015-2016 Vladimir Kondratyev <wulf@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,10 +31,11 @@
 #define	_DEV_EVDEV_EVDEV_PRIVATE_H
 
 #include <sys/bitstring.h>
-#include <sys/queue.h>
-#include <sys/malloc.h>
 #include <sys/kbio.h>
+#include <sys/malloc.h>
+#include <sys/queue.h>
 #include <sys/selinfo.h>
+
 #include <dev/evdev/evdev.h>
 #include <dev/evdev/input.h>
 #include <dev/kbd/kbdreg.h>
@@ -134,6 +135,16 @@ struct evdev_dev
 #define	EVDEV_LOCK(evdev)		mtx_lock((evdev)->ev_lock)
 #define	EVDEV_UNLOCK(evdev)		mtx_unlock((evdev)->ev_lock)
 #define	EVDEV_LOCK_ASSERT(evdev)	mtx_assert((evdev)->ev_lock, MA_OWNED)
+#define	EVDEV_ENTER(evdev)	do {					\
+	if ((evdev)->ev_lock_type == EV_LOCK_INTERNAL)			\
+		EVDEV_LOCK(evdev);					\
+	else								\
+		EVDEV_LOCK_ASSERT(evdev);				\
+} while (0)
+#define	EVDEV_EXIT(evdev)	do {					\
+	if ((evdev)->ev_lock_type == EV_LOCK_INTERNAL)			\
+		EVDEV_UNLOCK(evdev);					\
+} while (0)
 
 struct evdev_client
 {
diff --git a/freebsd/sys/dev/evdev/evdev_utils.c b/freebsd/sys/dev/evdev/evdev_utils.c
index 49e03b6a..caf81a46 100644
--- a/freebsd/sys/dev/evdev/evdev_utils.c
+++ b/freebsd/sys/dev/evdev/evdev_utils.c
@@ -2,7 +2,7 @@
 
 /*-
  * Copyright (c) 2014 Jakub Wojciech Klama <jceel@FreeBSD.org>
- * Copyright (c) 2015-2016 Vladimir Kondratyev <wulf@cicgroup.ru>
+ * Copyright (c) 2015-2016 Vladimir Kondratyev <wulf@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -29,17 +29,16 @@
  * $FreeBSD$
  */
 
-#include <sys/types.h>
-#include <sys/systm.h>
 #include <sys/param.h>
 #include <sys/bus.h>
-#include <sys/kernel.h>
 #include <sys/conf.h>
-#include <sys/malloc.h>
 #include <sys/kbio.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/systm.h>
 
-#include <dev/evdev/input.h>
 #include <dev/evdev/evdev.h>
+#include <dev/evdev/input.h>
 
 #include <dev/kbd/kbdreg.h>
 
diff --git a/freebsd/sys/dev/evdev/input-event-codes.h b/freebsd/sys/dev/evdev/input-event-codes.h
index 78ba7d2e..cc1528f6 100644
--- a/freebsd/sys/dev/evdev/input-event-codes.h
+++ b/freebsd/sys/dev/evdev/input-event-codes.h
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2016 Oleksandr Tymoshenko <gonzo@FreeBSD.org>
- * Copyright (c) 2015-2016 Vladimir Kondratyev <wulf@cicgroup.ru>
+ * Copyright (c) 2015-2016 Vladimir Kondratyev <wulf@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/freebsd/sys/dev/evdev/input.h b/freebsd/sys/dev/evdev/input.h
index 04638444..7639e0d6 100644
--- a/freebsd/sys/dev/evdev/input.h
+++ b/freebsd/sys/dev/evdev/input.h
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2016 Oleksandr Tymoshenko <gonzo@FreeBSD.org>
- * Copyright (c) 2015-2016 Vladimir Kondratyev <wulf@cicgroup.ru>
+ * Copyright (c) 2015-2016 Vladimir Kondratyev <wulf@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
 #define	_EVDEV_INPUT_H
 
 #ifndef __KERNEL__
-#include <sys/time.h>
 #include <sys/ioccom.h>
+#include <sys/time.h>
 #include <sys/types.h>
 #endif
 
diff --git a/freebsd/sys/dev/evdev/uinput.c b/freebsd/sys/dev/evdev/uinput.c
index efebf02a..f1f812cc 100644
--- a/freebsd/sys/dev/evdev/uinput.c
+++ b/freebsd/sys/dev/evdev/uinput.c
@@ -2,7 +2,7 @@
 
 /*-
  * Copyright (c) 2014 Jakub Wojciech Klama <jceel@FreeBSD.org>
- * Copyright (c) 2015-2016 Vladimir Kondratyev <wulf@cicgroup.ru>
+ * Copyright (c) 2015-2016 Vladimir Kondratyev <wulf@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,25 +31,24 @@
 
 #include <rtems/bsd/local/opt_evdev.h>
 
-#include <sys/types.h>
-#include <sys/systm.h>
 #include <sys/param.h>
+#include <sys/conf.h>
 #include <sys/fcntl.h>
 #include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
 #include <sys/module.h>
-#include <sys/conf.h>
-#include <sys/uio.h>
-#include <sys/proc.h>
 #include <sys/poll.h>
+#include <sys/proc.h>
 #include <sys/selinfo.h>
-#include <sys/malloc.h>
-#include <sys/lock.h>
+#include <sys/systm.h>
 #include <sys/sx.h>
+#include <sys/uio.h>
 
-#include <dev/evdev/input.h>
-#include <dev/evdev/uinput.h>
 #include <dev/evdev/evdev.h>
 #include <dev/evdev/evdev_private.h>
+#include <dev/evdev/input.h>
+#include <dev/evdev/uinput.h>
 
 #ifdef UINPUT_DEBUG
 #define	debugf(state, fmt, args...)	printf("uinput: " fmt "\n", ##args)
diff --git a/freebsd/sys/dev/evdev/uinput.h b/freebsd/sys/dev/evdev/uinput.h
index f1721e19..dd4b0a82 100644
--- a/freebsd/sys/dev/evdev/uinput.h
+++ b/freebsd/sys/dev/evdev/uinput.h
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2016 Oleksandr Tymoshenko <gonzo@FreeBSD.org>
- * Copyright (c) 2015-2016 Vladimir Kondratyev <wulf@cicgroup.ru>
+ * Copyright (c) 2015-2016 Vladimir Kondratyev <wulf@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/freebsd/sys/dev/ffec/if_ffec.c b/freebsd/sys/dev/ffec/if_ffec.c
index b9a0d668..22b2cdfc 100644
--- a/freebsd/sys/dev/ffec/if_ffec.c
+++ b/freebsd/sys/dev/ffec/if_ffec.c
@@ -90,6 +90,7 @@ __FBSDID("$FreeBSD$");
 #include <dev/ofw/ofw_bus_subr.h>
 #include <dev/mii/mii.h>
 #include <dev/mii/miivar.h>
+#include <dev/mii/mii_fdt.h>
 #include <rtems/bsd/local/miibus_if.h>
 
 /*
@@ -140,7 +141,6 @@ static struct ofw_compat_data compat_data[] = {
 #define	TX_MAX_DMA_SEGS	8
 
 #define	WATCHDOG_TIMEOUT_SECS	5
-#define	STATS_HARVEST_INTERVAL	3
 
 #define	MAX_IRQ_COUNT 3
 
@@ -153,13 +153,6 @@ struct ffec_bufmap {
 	bus_dmamap_t	map;
 };
 
-enum {
-	PHY_CONN_UNKNOWN,
-	PHY_CONN_MII,
-	PHY_CONN_RMII,
-	PHY_CONN_RGMII
-};
-
 struct ffec_softc {
 	device_t		dev;
 	device_t		miibus;
@@ -172,15 +165,14 @@ struct ffec_softc {
 	struct resource		*mem_res;
 	void *			intr_cookie[MAX_IRQ_COUNT];
 	struct callout		ffec_callout;
-	uint8_t			phy_conn_type;
+	mii_contype_t		phy_conn_type;
 	uintptr_t		fectype;
 	boolean_t		link_is_up;
 	boolean_t		is_attached;
 	boolean_t		is_detaching;
 	int			tx_watchdog_count;
-	int			stats_harvest_count;
-	int                     rxbuf_align;
-	int                     txbuf_align;
+	int			rxbuf_align;
+	int			txbuf_align;
 
 	bus_dma_tag_t		rxdesc_tag;
 	bus_dmamap_t		rxdesc_map;
@@ -320,10 +312,10 @@ ffec_miigasket_setup(struct ffec_softc *sc)
 
 	switch (sc->phy_conn_type)
 	{
-	case PHY_CONN_MII:
+	case MII_CONTYPE_MII:
 		ifmode = 0;
 		break;
-	case PHY_CONN_RMII:
+	case MII_CONTYPE_RMII:
 		ifmode = FEC_MIIGSK_CFGR_IF_MODE_RMII;
 		break;
 	default:
@@ -435,14 +427,17 @@ ffec_miibus_statchg(device_t dev)
 
 	rcr |= FEC_RCR_MII_MODE; /* Must always be on even for R[G]MII. */
 	switch (sc->phy_conn_type) {
-	case PHY_CONN_MII:
-		break;
-	case PHY_CONN_RMII:
+	case MII_CONTYPE_RMII:
 		rcr |= FEC_RCR_RMII_MODE;
 		break;
-	case PHY_CONN_RGMII:
+	case MII_CONTYPE_RGMII:
+	case MII_CONTYPE_RGMII_ID:
+	case MII_CONTYPE_RGMII_RXID:
+	case MII_CONTYPE_RGMII_TXID:
 		rcr |= FEC_RCR_RGMII_EN;
 		break;
+	default:
+		break;
 	}
 
 	switch (IFM_SUBTYPE(mii->mii_media_active)) {
@@ -518,22 +513,41 @@ ffec_media_change(struct ifnet * ifp)
 
 static void ffec_clear_stats(struct ffec_softc *sc)
 {
+	uint32_t mibc;
 
-	WR4(sc, FEC_RMON_R_PACKETS, 0);
-	WR4(sc, FEC_RMON_R_MC_PKT, 0);
-	WR4(sc, FEC_RMON_R_CRC_ALIGN, 0);
-	WR4(sc, FEC_RMON_R_UNDERSIZE, 0);
-	WR4(sc, FEC_RMON_R_OVERSIZE, 0);
-	WR4(sc, FEC_RMON_R_FRAG, 0);
-	WR4(sc, FEC_RMON_R_JAB, 0);
-	WR4(sc, FEC_RMON_T_PACKETS, 0);
-	WR4(sc, FEC_RMON_T_MC_PKT, 0);
-	WR4(sc, FEC_RMON_T_CRC_ALIGN, 0);
-	WR4(sc, FEC_RMON_T_UNDERSIZE, 0);
-	WR4(sc, FEC_RMON_T_OVERSIZE , 0);
-	WR4(sc, FEC_RMON_T_FRAG, 0);
-	WR4(sc, FEC_RMON_T_JAB, 0);
-	WR4(sc, FEC_RMON_T_COL, 0);
+	mibc = RD4(sc, FEC_MIBC_REG);
+
+	/*
+	 * On newer hardware the statistic regs are cleared by toggling a bit in
+	 * the mib control register.  On older hardware the clear procedure is
+	 * to disable statistics collection, zero the regs, then re-enable.
+	 */
+	if (sc->fectype == FECTYPE_IMX6 || sc->fectype == FECTYPE_MVF) {
+		WR4(sc, FEC_MIBC_REG, mibc | FEC_MIBC_CLEAR);
+		WR4(sc, FEC_MIBC_REG, mibc & ~FEC_MIBC_CLEAR);
+	} else {
+		WR4(sc, FEC_MIBC_REG, mibc | FEC_MIBC_DIS);
+	
+		WR4(sc, FEC_IEEE_R_DROP, 0);
+		WR4(sc, FEC_IEEE_R_MACERR, 0);
+		WR4(sc, FEC_RMON_R_CRC_ALIGN, 0);
+		WR4(sc, FEC_RMON_R_FRAG, 0);
+		WR4(sc, FEC_RMON_R_JAB, 0);
+		WR4(sc, FEC_RMON_R_MC_PKT, 0);
+		WR4(sc, FEC_RMON_R_OVERSIZE, 0);
+		WR4(sc, FEC_RMON_R_PACKETS, 0);
+		WR4(sc, FEC_RMON_R_UNDERSIZE, 0);
+		WR4(sc, FEC_RMON_T_COL, 0);
+		WR4(sc, FEC_RMON_T_CRC_ALIGN, 0);
+		WR4(sc, FEC_RMON_T_FRAG, 0);
+		WR4(sc, FEC_RMON_T_JAB, 0);
+		WR4(sc, FEC_RMON_T_MC_PKT, 0);
+		WR4(sc, FEC_RMON_T_OVERSIZE , 0);
+		WR4(sc, FEC_RMON_T_PACKETS, 0);
+		WR4(sc, FEC_RMON_T_UNDERSIZE, 0);
+
+		WR4(sc, FEC_MIBC_REG, mibc);
+	}
 }
 
 static void
@@ -541,28 +555,21 @@ ffec_harvest_stats(struct ffec_softc *sc)
 {
 	struct ifnet *ifp;
 
-	/* We don't need to harvest too often. */
-	if (++sc->stats_harvest_count < STATS_HARVEST_INTERVAL)
-		return;
+	ifp = sc->ifp;
 
 	/*
-	 * Try to avoid harvesting unless the IDLE flag is on, but if it has
-	 * been too long just go ahead and do it anyway, the worst that'll
-	 * happen is we'll lose a packet count or two as we clear at the end.
+	 * - FEC_IEEE_R_DROP is "dropped due to invalid start frame delimiter"
+	 *   so it's really just another type of input error.
+	 * - FEC_IEEE_R_MACERR is "no receive fifo space"; count as input drops.
 	 */
-	if (sc->stats_harvest_count < (2 * STATS_HARVEST_INTERVAL) &&
-	    ((RD4(sc, FEC_MIBC_REG) & FEC_MIBC_IDLE) == 0))
-		return;
-
-	sc->stats_harvest_count = 0;
-	ifp = sc->ifp;
-
 	if_inc_counter(ifp, IFCOUNTER_IPACKETS, RD4(sc, FEC_RMON_R_PACKETS));
 	if_inc_counter(ifp, IFCOUNTER_IMCASTS, RD4(sc, FEC_RMON_R_MC_PKT));
 	if_inc_counter(ifp, IFCOUNTER_IERRORS,
 	    RD4(sc, FEC_RMON_R_CRC_ALIGN) + RD4(sc, FEC_RMON_R_UNDERSIZE) +
 	    RD4(sc, FEC_RMON_R_OVERSIZE) + RD4(sc, FEC_RMON_R_FRAG) +
-	    RD4(sc, FEC_RMON_R_JAB));
+	    RD4(sc, FEC_RMON_R_JAB) + RD4(sc, FEC_IEEE_R_DROP));
+
+	if_inc_counter(ifp, IFCOUNTER_IQDROPS, RD4(sc, FEC_IEEE_R_MACERR));
 
 	if_inc_counter(ifp, IFCOUNTER_OPACKETS, RD4(sc, FEC_RMON_T_PACKETS));
 	if_inc_counter(ifp, IFCOUNTER_OMCASTS, RD4(sc, FEC_RMON_T_MC_PKT));
@@ -1132,7 +1139,6 @@ ffec_stop_locked(struct ffec_softc *sc)
 	ifp = sc->ifp;
 	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
 	sc->tx_watchdog_count = 0;
-	sc->stats_harvest_count = 0;
 
 	/* 
 	 * Stop the hardware, mask all interrupts, and clear all current
@@ -1311,7 +1317,8 @@ ffec_init_locked(struct ffec_softc *sc)
 	WR4(sc, FEC_IEM_REG, FEC_IER_TXF | FEC_IER_RXF | FEC_IER_EBERR);
 
 	/*
-	 * MIBC - MIB control (hardware stats).
+	 * MIBC - MIB control (hardware stats); clear all statistics regs, then
+	 * enable collection of statistics.
 	 */
 	regval = RD4(sc, FEC_MIBC_REG);
 	WR4(sc, FEC_MIBC_REG, regval | FEC_MIBC_DIS);
@@ -1708,7 +1715,6 @@ ffec_attach(device_t dev)
 	phandle_t ofw_node;
 	int error, phynum, rid, irq;
 	uint8_t eaddr[ETHER_ADDR_LEN];
-	char phy_conn_name[32];
 	uint32_t idx, mscr;
 
 	sc = device_get_softc(dev);
@@ -1739,20 +1745,8 @@ ffec_attach(device_t dev)
 		error = ENXIO;
 		goto out;
 	}
-	if (OF_searchprop(ofw_node, "phy-mode", 
-	    phy_conn_name, sizeof(phy_conn_name)) != -1) {
-		if (strcasecmp(phy_conn_name, "mii") == 0)
-			sc->phy_conn_type = PHY_CONN_MII;
-		else if (strcasecmp(phy_conn_name, "rmii") == 0)
-			sc->phy_conn_type = PHY_CONN_RMII;
-#ifndef __rtems__
-		else if (strcasecmp(phy_conn_name, "rgmii") == 0)
-#else /* __rtems__ */
-		else if (strncasecmp(phy_conn_name, "rgmii", 5) == 0)
-#endif /* __rtems__ */
-			sc->phy_conn_type = PHY_CONN_RGMII;
-	}
-	if (sc->phy_conn_type == PHY_CONN_UNKNOWN) {
+	sc->phy_conn_type = mii_fdt_get_contype(ofw_node);
+	if (sc->phy_conn_type == MII_CONTYPE_UNKNOWN) {
 		device_printf(sc->dev, "No valid 'phy-mode' "
 		    "property found in FDT data for device.\n");
 #ifndef __rtems__
diff --git a/freebsd/sys/dev/mii/mii_fdt.c b/freebsd/sys/dev/mii/mii_fdt.c
new file mode 100644
index 00000000..de994ff9
--- /dev/null
+++ b/freebsd/sys/dev/mii/mii_fdt.c
@@ -0,0 +1,202 @@
+#include <machine/rtems-bsd-kernel-space.h>
+
+/*-
+ * Copyright (c) 2017 Ian Lepore <ian@freebsd.org>
+ * All rights reserved.
+ *
+ * Development sponsored by Microsemi, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+/*
+ * Utility functions for PHY drivers on systems configured using FDT data.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/socket.h>
+#include <sys/bus.h>
+#include <sys/malloc.h>
+
+#include <net/if.h>
+#include <net/if_media.h>
+
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include <dev/mii/mii.h>
+#include <dev/mii/miivar.h>
+#include <dev/mii/mii_fdt.h>
+
+/*
+ * Table to translate MII_CONTYPE_xxxx constants to/from devicetree strings.
+ * We explicitly associate the enum values with the strings in a table to avoid
+ * relying on this list being sorted in the same order as the enum in miivar.h,
+ * and to avoid problems if the enum gains new types that aren't in the FDT
+ * data.  However, the "unknown" entry must be first because it is referenced
+ * using subscript 0 in mii_fdt_contype_to_name().
+ */
+static struct contype_names {
+	mii_contype_t type;
+	const char   *name;
+} fdt_contype_names[] = {
+	{MII_CONTYPE_UNKNOWN,		"unknown"},
+	{MII_CONTYPE_MII,		"mii"},
+	{MII_CONTYPE_GMII,		"gmii"},
+	{MII_CONTYPE_SGMII,		"sgmii"},
+	{MII_CONTYPE_QSGMII,		"qsgmii"},
+	{MII_CONTYPE_TBI,		"tbi"},
+	{MII_CONTYPE_REVMII,		"rev-mii"},
+	{MII_CONTYPE_RMII,		"rmii"},
+	{MII_CONTYPE_RGMII,		"rgmii"},
+	{MII_CONTYPE_RGMII_ID,		"rgmii-id"},
+	{MII_CONTYPE_RGMII_RXID,	"rgmii-rxid"},
+	{MII_CONTYPE_RGMII_TXID,	"rgmii-txid"},
+	{MII_CONTYPE_RTBI,		"rtbi"},
+	{MII_CONTYPE_SMII,		"smii"},
+	{MII_CONTYPE_XGMII,		"xgmii"},
+	{MII_CONTYPE_TRGMII,		"trgmii"},
+	{MII_CONTYPE_2000BX,		"2000base-x"},
+	{MII_CONTYPE_2500BX,		"2500base-x"},
+	{MII_CONTYPE_RXAUI,		"rxaui"},
+};                                                           
+
+static phandle_t
+mii_fdt_get_phynode(phandle_t macnode)
+{
+	static const char *props[] = {
+	    "phy-handle", "phy", "phy-device"
+	};
+	pcell_t xref;
+	u_int i;
+
+	for (i = 0; i < nitems(props); ++i) {
+		if (OF_getencprop(macnode, props[i], &xref, sizeof(xref)) > 0)
+			return (OF_node_from_xref(xref));
+	}
+	return (-1);
+}
+
+mii_contype_t
+mii_fdt_contype_from_name(const char *name)
+{
+	u_int i;
+
+	for (i = 0; i < nitems(fdt_contype_names); ++i) {
+		if (strcmp(name, fdt_contype_names[i].name) == 0)
+			return (fdt_contype_names[i].type);
+	}
+	return (MII_CONTYPE_UNKNOWN);
+}
+
+const char *
+mii_fdt_contype_to_name(mii_contype_t contype)
+{
+	u_int i;
+
+	for (i = 0; i < nitems(fdt_contype_names); ++i) {
+		if (contype == fdt_contype_names[i].type)
+			return (fdt_contype_names[i].name);
+	}
+	return (fdt_contype_names[0].name);
+}
+
+mii_contype_t
+mii_fdt_get_contype(phandle_t macnode)
+{
+	char val[32];
+
+	if (OF_getprop(macnode, "phy-mode", val, sizeof(val)) <= 0 &&
+	    OF_getprop(macnode, "phy-connection-type", val, sizeof(val)) <= 0) {
+                return (MII_CONTYPE_UNKNOWN);
+	}
+	return (mii_fdt_contype_from_name(val));
+}
+
+void
+mii_fdt_free_config(struct mii_fdt_phy_config *cfg)
+{
+
+	free(cfg, M_OFWPROP);
+}
+
+mii_fdt_phy_config_t *
+mii_fdt_get_config(device_t phydev)
+{
+	mii_fdt_phy_config_t *cfg;
+	device_t miibus, macdev;
+	pcell_t val;
+
+	miibus = device_get_parent(phydev);
+	macdev = device_get_parent(miibus);
+
+	cfg = malloc(sizeof(*cfg), M_OFWPROP, M_ZERO | M_WAITOK);
+
+	/*
+	 * If we can't find our parent MAC's node, there's nothing more we can
+	 * fill in; cfg is already full of zero/default values, return it.
+	 */
+	if ((cfg->macnode = ofw_bus_get_node(macdev)) == -1)
+		return (cfg);
+
+	cfg->con_type = mii_fdt_get_contype(cfg->macnode);
+
+	/*
+	 * If we can't find our own PHY node, there's nothing more we can fill
+	 * in, just return what we've got.
+	 */
+	if ((cfg->phynode = mii_fdt_get_phynode(cfg->macnode)) == -1)
+		return (cfg);
+
+	if (OF_getencprop(cfg->phynode, "max-speed", &val, sizeof(val)) > 0)
+		cfg->max_speed = val;
+
+	if (ofw_bus_node_is_compatible(cfg->phynode,
+	    "ethernet-phy-ieee802.3-c45"))
+		cfg->flags |= MIIF_FDT_COMPAT_CLAUSE45;
+
+	if (OF_hasprop(cfg->phynode, "broken-turn-around"))
+		cfg->flags |= MIIF_FDT_BROKEN_TURNAROUND;
+	if (OF_hasprop(cfg->phynode, "enet-phy-lane-swap"))
+		cfg->flags |= MIIF_FDT_LANE_SWAP;
+	if (OF_hasprop(cfg->phynode, "enet-phy-lane-no-swap"))
+		cfg->flags |= MIIF_FDT_NO_LANE_SWAP;
+	if (OF_hasprop(cfg->phynode, "eee-broken-100tx"))
+		cfg->flags |= MIIF_FDT_EEE_BROKEN_100TX;
+	if (OF_hasprop(cfg->phynode, "eee-broken-1000t"))
+		cfg->flags |= MIIF_FDT_EEE_BROKEN_1000T;
+	if (OF_hasprop(cfg->phynode, "eee-broken-10gt"))
+		cfg->flags |= MIIF_FDT_EEE_BROKEN_10GT;
+	if (OF_hasprop(cfg->phynode, "eee-broken-1000kx"))
+		cfg->flags |= MIIF_FDT_EEE_BROKEN_1000KX;
+	if (OF_hasprop(cfg->phynode, "eee-broken-10gkx4"))
+		cfg->flags |= MIIF_FDT_EEE_BROKEN_10GKX4;
+	if (OF_hasprop(cfg->phynode, "eee-broken-10gkr"))
+		cfg->flags |= MIIF_FDT_EEE_BROKEN_10GKR;
+
+	return (cfg);
+}
diff --git a/freebsd/sys/dev/mii/mii_fdt.h b/freebsd/sys/dev/mii/mii_fdt.h
new file mode 100644
index 00000000..7b0b5131
--- /dev/null
+++ b/freebsd/sys/dev/mii/mii_fdt.h
@@ -0,0 +1,75 @@
+/*-
+ * Copyright (c) 2017 Ian Lepore <ian@freebsd.org>
+ * All rights reserved.
+ *
+ * Development sponsored by Microsemi, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _DEV_MII_FDT_H_
+#define	_DEV_MII_FDT_H_
+
+/*
+ * Common FDT config for a PHY, as documented in the devicetree bindings
+ * documents ethernet.txt and phy.txt.  Boolean properties are represented as
+ * bits in the flags member.
+ */
+struct mii_fdt_phy_config {
+	phandle_t	macnode;	/* Node (not xref) of parent MAC */
+	phandle_t	phynode;	/* Node (not xref) of PHY */
+	mii_contype_t	con_type;	/* MAC<->PHY connection type */
+	u_int		max_speed;	/* Mbits/sec, 0 = not specified */
+	uint32_t	flags;		/* MIIF_FDT_xxx boolean properties */
+};
+typedef struct mii_fdt_phy_config mii_fdt_phy_config_t;
+
+/* PHY config flags. */
+#define	MIIF_FDT_COMPAT_CLAUSE45	0x0001
+#define	MIIF_FDT_BROKEN_TURNAROUND	0x0002
+#define	MIIF_FDT_LANE_SWAP		0x0004
+#define	MIIF_FDT_NO_LANE_SWAP		0x0008
+#define	MIIF_FDT_EEE_BROKEN_100TX	0x0010
+#define	MIIF_FDT_EEE_BROKEN_1000T	0x0020
+#define	MIIF_FDT_EEE_BROKEN_10GT	0x0040
+#define	MIIF_FDT_EEE_BROKEN_1000KX	0x0080
+#define	MIIF_FDT_EEE_BROKEN_10GKX4	0x0100
+#define	MIIF_FDT_EEE_BROKEN_10GKR	0x0200
+
+/*
+ * Convert between mii_contype enums and devicetree property strings.
+ */
+const char *mii_fdt_contype_to_name(mii_contype_t contype);
+mii_contype_t mii_fdt_contype_from_name(const char *name);
+
+/* Get the connection type from the given MAC node. */
+mii_contype_t mii_fdt_get_contype(phandle_t macnode);
+
+/*
+ * Get/free the config for the given PHY device.
+ */
+void mii_fdt_free_config(struct mii_fdt_phy_config *cfg);
+mii_fdt_phy_config_t *mii_fdt_get_config(device_t phydev);
+
+#endif
diff --git a/freebsd/sys/dev/mii/miivar.h b/freebsd/sys/dev/mii/miivar.h
index 498e7204..ef81bdb2 100644
--- a/freebsd/sys/dev/mii/miivar.h
+++ b/freebsd/sys/dev/mii/miivar.h
@@ -156,6 +156,42 @@ typedef struct mii_softc mii_softc_t;
 #define	MII_PHY_ANY		-1
 
 /*
+ * Constants used to describe the type of attachment between MAC and PHY.
+ */
+enum mii_contype {
+	MII_CONTYPE_UNKNOWN,	/* Must be have value 0. */
+
+	MII_CONTYPE_MII,
+	MII_CONTYPE_GMII,
+	MII_CONTYPE_SGMII,
+	MII_CONTYPE_QSGMII,
+	MII_CONTYPE_TBI,
+	MII_CONTYPE_REVMII,	/* Reverse MII */
+	MII_CONTYPE_RMII,
+	MII_CONTYPE_RGMII,	/* Delays provided by MAC or PCB */
+	MII_CONTYPE_RGMII_ID,	/* Rx and tx delays provided by PHY */
+	MII_CONTYPE_RGMII_RXID,	/* Only rx delay provided by PHY */
+	MII_CONTYPE_RGMII_TXID,	/* Only tx delay provided by PHY */
+	MII_CONTYPE_RTBI,
+	MII_CONTYPE_SMII,
+	MII_CONTYPE_XGMII,
+	MII_CONTYPE_TRGMII,
+	MII_CONTYPE_2000BX,
+	MII_CONTYPE_2500BX,
+	MII_CONTYPE_RXAUI,
+
+	MII_CONTYPE_COUNT	/* Add new types before this line. */
+};
+typedef enum mii_contype mii_contype_t;
+
+static inline bool
+mii_contype_is_rgmii(mii_contype_t con)
+{
+
+	return (con >= MII_CONTYPE_RGMII && con <= MII_CONTYPE_RGMII_TXID);
+}
+
+/*
  * Used to attach a PHY to a parent.
  */
 struct mii_attach_args {
diff --git a/freebsd/sys/dev/mmc/bridge.h b/freebsd/sys/dev/mmc/bridge.h
index a780ffae..53e61b48 100644
--- a/freebsd/sys/dev/mmc/bridge.h
+++ b/freebsd/sys/dev/mmc/bridge.h
@@ -137,6 +137,10 @@ enum mmc_card_mode {
 	mode_mmc, mode_sd
 };
 
+enum mmc_retune_req {
+	retune_req_none = 0, retune_req_normal, retune_req_reset
+};
+
 struct mmc_host {
 	int f_min;
 	int f_max;
@@ -174,15 +178,17 @@ struct mmc_host {
 	struct mmc_ios ios;	/* Current state of the host */
 };
 
+#ifdef _KERNEL
 extern driver_t   mmc_driver;
 extern devclass_t mmc_devclass;
 
-#define	MMC_VERSION	3
+#define	MMC_VERSION	4
 
 #define	MMC_DECLARE_BRIDGE(name)					\
     DRIVER_MODULE(mmc, name, mmc_driver, mmc_devclass, NULL, NULL);	\
     MODULE_DEPEND(name, mmc, MMC_VERSION, MMC_VERSION, MMC_VERSION);
 #define	MMC_DEPEND(name)						\
     MODULE_DEPEND(name, mmc, MMC_VERSION, MMC_VERSION, MMC_VERSION);
+#endif /* _KERNEL */
 
 #endif /* DEV_MMC_BRIDGE_H */
diff --git a/freebsd/sys/dev/mmc/mmc.c b/freebsd/sys/dev/mmc/mmc.c
index 023091eb..74e26332 100644
--- a/freebsd/sys/dev/mmc/mmc.c
+++ b/freebsd/sys/dev/mmc/mmc.c
@@ -90,14 +90,14 @@ struct mmc_ivars {
 	uint8_t raw_ext_csd[MMC_EXTCSD_SIZE]; /* Raw bits of the EXT_CSD */
 	uint32_t raw_sd_status[16];	/* Raw bits of the SD_STATUS */
 	uint16_t rca;
+	u_char read_only;	/* True when the device is read-only */
+	u_char high_cap;	/* High Capacity device (block addressed) */
 	enum mmc_card_mode mode;
+	enum mmc_bus_width bus_width;	/* Bus width to use */
 	struct mmc_cid cid;	/* cid decoded */
 	struct mmc_csd csd;	/* csd decoded */
 	struct mmc_scr scr;	/* scr decoded */
 	struct mmc_sd_status sd_status;	/* SD_STATUS decoded */
-	u_char read_only;	/* True when the device is read-only */
-	u_char bus_width;	/* Bus width to use */
-	u_char high_cap;	/* High Capacity card (block addressed) */
 	uint32_t sec_count;	/* Card capacity in 512byte blocks */
 	uint32_t timings;	/* Mask of bus timings supported */
 	uint32_t vccq_120;	/* Mask of bus timings at VCCQ of 1.2 V */
@@ -129,8 +129,10 @@ static int mmc_read_ivar(device_t bus, device_t child, int which,
     uintptr_t *result);
 static int mmc_release_bus(device_t busdev, device_t dev);
 static int mmc_resume(device_t dev);
+static void mmc_retune_pause(device_t busdev, device_t dev, bool retune);
+static void mmc_retune_unpause(device_t busdev, device_t dev);
 static int mmc_suspend(device_t dev);
-static int mmc_wait_for_request(device_t brdev, device_t reqdev,
+static int mmc_wait_for_request(device_t busdev, device_t dev,
     struct mmc_request *req);
 static int mmc_write_ivar(device_t bus, device_t child, int which,
     uintptr_t value);
@@ -157,21 +159,23 @@ static void mmc_decode_cid_mmc(uint32_t *raw_cid, struct mmc_cid *cid,
     bool is_4_41p);
 static void mmc_decode_cid_sd(uint32_t *raw_cid, struct mmc_cid *cid);
 static void mmc_decode_csd_mmc(uint32_t *raw_csd, struct mmc_csd *csd);
-static void mmc_decode_csd_sd(uint32_t *raw_csd, struct mmc_csd *csd);
+static int mmc_decode_csd_sd(uint32_t *raw_csd, struct mmc_csd *csd);
 static void mmc_delayed_attach(void *xsc);
-static int mmc_delete_cards(struct mmc_softc *sc);
+static int mmc_delete_cards(struct mmc_softc *sc, bool final);
 static void mmc_discover_cards(struct mmc_softc *sc);
 static void mmc_format_card_id_string(struct mmc_ivars *ivar);
 static void mmc_go_discovery(struct mmc_softc *sc);
 static uint32_t mmc_get_bits(uint32_t *bits, int bit_len, int start,
     int size);
 static int mmc_highest_voltage(uint32_t ocr);
+static bool mmc_host_timing(device_t dev, enum mmc_bus_timing timing);
 static void mmc_idle_cards(struct mmc_softc *sc);
 static void mmc_ms_delay(int ms);
 static void mmc_log_card(device_t dev, struct mmc_ivars *ivar, int newcard);
 static void mmc_power_down(struct mmc_softc *sc);
 static void mmc_power_up(struct mmc_softc *sc);
 static void mmc_rescan_cards(struct mmc_softc *sc);
+static int mmc_retune(device_t busdev, device_t dev, bool reset);
 static void mmc_scan(struct mmc_softc *sc);
 static int mmc_sd_switch(struct mmc_softc *sc, uint8_t mode, uint8_t grp,
     uint8_t value, uint8_t *res);
@@ -185,15 +189,23 @@ static int mmc_send_op_cond(struct mmc_softc *sc, uint32_t ocr,
     uint32_t *rocr);
 static int mmc_send_relative_addr(struct mmc_softc *sc, uint32_t *resp);
 static int mmc_set_blocklen(struct mmc_softc *sc, uint32_t len);
-static int mmc_set_card_bus_width(struct mmc_softc *sc, struct mmc_ivars *ivar);
+static int mmc_set_card_bus_width(struct mmc_softc *sc, struct mmc_ivars *ivar,
+    enum mmc_bus_timing timing);
 static int mmc_set_power_class(struct mmc_softc *sc, struct mmc_ivars *ivar);
 static int mmc_set_relative_addr(struct mmc_softc *sc, uint16_t resp);
 static int mmc_set_timing(struct mmc_softc *sc, struct mmc_ivars *ivar,
     enum mmc_bus_timing timing);
+static int mmc_set_vccq(struct mmc_softc *sc, struct mmc_ivars *ivar,
+    enum mmc_bus_timing timing);
+static int mmc_switch_to_hs200(struct mmc_softc *sc, struct mmc_ivars *ivar,
+    uint32_t clock);
+static int mmc_switch_to_hs400(struct mmc_softc *sc, struct mmc_ivars *ivar,
+    uint32_t max_dtr, enum mmc_bus_timing max_timing);
 static int mmc_test_bus_width(struct mmc_softc *sc);
 static uint32_t mmc_timing_to_dtr(struct mmc_ivars *ivar,
     enum mmc_bus_timing timing);
 static const char *mmc_timing_to_string(enum mmc_bus_timing timing);
+static void mmc_update_child_list(struct mmc_softc *sc);
 static int mmc_wait_for_command(struct mmc_softc *sc, uint32_t opcode,
     uint32_t arg, uint32_t flags, uint32_t *resp, int retries);
 static int mmc_wait_for_req(struct mmc_softc *sc, struct mmc_request *req);
@@ -237,7 +249,8 @@ mmc_detach(device_t dev)
 	struct mmc_softc *sc = device_get_softc(dev);
 	int err;
 
-	if ((err = mmc_delete_cards(sc)) != 0)
+	err = mmc_delete_cards(sc, true);
+	if (err != 0)
 		return (err);
 	mmc_power_down(sc);
 	MMC_LOCK_DESTROY(sc);
@@ -252,10 +265,21 @@ mmc_suspend(device_t dev)
 	int err;
 
 	err = bus_generic_suspend(dev);
-	if (err)
+	if (err != 0)
+		return (err);
+	/*
+	 * We power down with the bus acquired here, mainly so that no device
+	 * is selected any longer and sc->last_rca gets set to 0.  Otherwise,
+	 * the deselect as part of the bus acquisition in mmc_scan() may fail
+	 * during resume, as the bus isn't powered up again before later in
+	 * mmc_go_discovery().
+	 */
+	err = mmc_acquire_bus(dev, dev);
+	if (err != 0)
 		return (err);
 	mmc_power_down(sc);
-	return (0);
+	err = mmc_release_bus(dev, dev);
+	return (err);
 }
 
 static int
@@ -272,7 +296,8 @@ mmc_acquire_bus(device_t busdev, device_t dev)
 {
 	struct mmc_softc *sc;
 	struct mmc_ivars *ivar;
-	int err, rca;
+	int err;
+	uint16_t rca;
 	enum mmc_bus_timing timing;
 
 	err = MMCBR_ACQUIRE_HOST(device_get_parent(busdev), busdev);
@@ -296,12 +321,27 @@ mmc_acquire_bus(device_t busdev, device_t dev)
 		rca = ivar->rca;
 		if (sc->last_rca != rca) {
 			if (mmc_select_card(sc, rca) != MMC_ERR_NONE) {
-				device_printf(sc->dev, "Card at relative "
-				    "address %d failed to select.\n", rca);
+				device_printf(busdev, "Card at relative "
+				    "address %d failed to select\n", rca);
 				return (ENXIO);
 			}
 			sc->last_rca = rca;
 			timing = mmcbr_get_timing(busdev);
+			/*
+			 * For eMMC modes, setting/updating bus width and VCCQ
+			 * only really is necessary if there actually is more
+			 * than one device on the bus as generally that already
+			 * had to be done by mmc_calculate_clock() or one of
+			 * its calees.  Moreover, setting the bus width anew
+			 * can trigger re-tuning (via a CRC error on the next
+			 * CMD), even if not switching between devices an the
+			 * previously selected one is still tuned.  Obviously,
+			 * we need to re-tune the host controller if devices
+			 * are actually switched, though.
+			 */
+			if (timing >= bus_timing_mmc_ddr52 &&
+			    sc->child_count == 1)
+				return (0);
 			/* Prepare bus width for the new card. */
 			if (bootverbose || mmc_debug) {
 				device_printf(busdev,
@@ -310,38 +350,34 @@ mmc_acquire_bus(device_t busdev, device_t dev)
 				    (ivar->bus_width == bus_width_8) ? 8 : 1,
 				    mmc_timing_to_string(timing));
 			}
-			if (mmc_set_card_bus_width(sc, ivar) != MMC_ERR_NONE) {
-				device_printf(sc->dev, "Card at relative "
-				    "address %d failed to set bus width.\n",
+			if (mmc_set_card_bus_width(sc, ivar, timing) !=
+			    MMC_ERR_NONE) {
+				device_printf(busdev, "Card at relative "
+				    "address %d failed to set bus width\n",
 				    rca);
 				return (ENXIO);
 			}
-			if (isset(&ivar->vccq_120, timing))
-				mmcbr_set_vccq(busdev, vccq_120);
-			else if (isset(&ivar->vccq_180, timing))
-				mmcbr_set_vccq(busdev, vccq_180);
-			else
-				mmcbr_set_vccq(busdev, vccq_330);
-			if (mmcbr_switch_vccq(busdev) != 0) {
-				device_printf(sc->dev, "Failed to set VCCQ "
-				    "for card at relative address %d.\n", rca);
+			mmcbr_set_bus_width(busdev, ivar->bus_width);
+			mmcbr_update_ios(busdev);
+			if (mmc_set_vccq(sc, ivar, timing) != MMC_ERR_NONE) {
+				device_printf(busdev, "Failed to set VCCQ "
+				    "for card at relative address %d\n", rca);
 				return (ENXIO);
 			}
-			if (mmc_set_power_class(sc, ivar) != MMC_ERR_NONE) {
-				device_printf(sc->dev, "Card at relative "
-				    "address %d failed to set power class.\n",
-				    rca);
+			if (timing >= bus_timing_mmc_hs200 &&
+			    mmc_retune(busdev, dev, true) != 0) {
+				device_printf(busdev, "Card at relative "
+				    "address %d failed to re-tune\n", rca);
 				return (ENXIO);
 			}
-			mmcbr_set_bus_width(busdev, ivar->bus_width);
-			mmcbr_update_ios(busdev);
 		}
 	} else {
 		/*
 		 * If there's a card selected, stand down.
 		 */
 		if (sc->last_rca != 0) {
-			mmc_select_card(sc, 0);
+			if (mmc_select_card(sc, 0) != MMC_ERR_NONE)
+				return (ENXIO);
 			sc->last_rca = 0;
 		}
 	}
@@ -416,7 +452,7 @@ mmc_wait_for_req(struct mmc_softc *sc, struct mmc_request *req)
 #endif /* __rtems__ */
 	req->done = mmc_wakeup;
 	req->done_data = sc;
-	if (mmc_debug > 1) {
+	if (__predict_false(mmc_debug > 1)) {
 		device_printf(sc->dev, "REQUEST: CMD%d arg %#x flags %#x",
 		    req->cmd->opcode, req->cmd->arg, req->cmd->flags);
 		if (req->cmd->data) {
@@ -434,18 +470,66 @@ mmc_wait_for_req(struct mmc_softc *sc, struct mmc_request *req)
 	rtems_binary_semaphore_wait(&req->req_done);
 	rtems_binary_semaphore_destroy(&req->req_done);
 #endif /* __rtems__ */
-	if (mmc_debug > 2 || (mmc_debug > 0 && req->cmd->error != MMC_ERR_NONE))
+	if (__predict_false(mmc_debug > 2 || (mmc_debug > 0 &&
+	    req->cmd->error != MMC_ERR_NONE)))
 		device_printf(sc->dev, "CMD%d RESULT: %d\n",
 		    req->cmd->opcode, req->cmd->error);
 	return (0);
 }
 
 static int
-mmc_wait_for_request(device_t brdev, device_t reqdev __unused,
-    struct mmc_request *req)
+mmc_wait_for_request(device_t busdev, device_t dev, struct mmc_request *req)
 {
-	struct mmc_softc *sc = device_get_softc(brdev);
+	struct mmc_softc *sc;
+	struct mmc_ivars *ivar;
+	int err, i;
+	enum mmc_retune_req retune_req;
+
+	sc = device_get_softc(busdev);
+	KASSERT(sc->owner != NULL,
+	    ("%s: Request from %s without bus being acquired.", __func__,
+	    device_get_nameunit(dev)));
 
+	/*
+	 * Unless no device is selected or re-tuning is already ongoing,
+	 * execute re-tuning if a) the bridge is requesting to do so and
+	 * re-tuning hasn't been otherwise paused, or b) if a child asked
+	 * to be re-tuned prior to pausing (see also mmc_retune_pause()).
+	 */
+	if (__predict_false(sc->last_rca != 0 && sc->retune_ongoing == 0 &&
+	    (((retune_req = mmcbr_get_retune_req(busdev)) != retune_req_none &&
+	    sc->retune_paused == 0) || sc->retune_needed == 1))) {
+		if (__predict_false(mmc_debug > 1)) {
+			device_printf(busdev,
+			    "Re-tuning with%s circuit reset required\n",
+			    retune_req == retune_req_reset ? "" : "out");
+		}
+		if (device_get_parent(dev) == busdev)
+			ivar = device_get_ivars(dev);
+		else {
+			for (i = 0; i < sc->child_count; i++) {
+				ivar = device_get_ivars(sc->child_list[i]);
+				if (ivar->rca == sc->last_rca)
+					break;
+			}
+			if (ivar->rca != sc->last_rca)
+				return (EINVAL);
+		}
+		sc->retune_ongoing = 1;
+		err = mmc_retune(busdev, dev, retune_req == retune_req_reset);
+		sc->retune_ongoing = 0;
+		switch (err) {
+		case MMC_ERR_NONE:
+		case MMC_ERR_FAILED:	/* Re-tune error but still might work */
+			break;
+		case MMC_ERR_BADCRC:	/* Switch failure on HS400 recovery */
+			return (ENXIO);
+		case MMC_ERR_INVALID:	/* Driver implementation b0rken */
+		default:		/* Unknown error, should not happen */
+			return (EINVAL);
+		}
+		sc->retune_needed = 0;
+	}
 	return (mmc_wait_for_req(sc, req));
 }
 
@@ -613,11 +697,14 @@ mmc_power_down(struct mmc_softc *sc)
 static int
 mmc_select_card(struct mmc_softc *sc, uint16_t rca)
 {
-	int flags;
+	int err, flags;
 
 	flags = (rca ? MMC_RSP_R1B : MMC_RSP_NONE) | MMC_CMD_AC;
-	return (mmc_wait_for_command(sc, MMC_SELECT_CARD, (uint32_t)rca << 16,
-	    flags, NULL, CMD_RETRIES));
+	sc->retune_paused++;
+	err = mmc_wait_for_command(sc, MMC_SELECT_CARD, (uint32_t)rca << 16,
+	    flags, NULL, CMD_RETRIES);
+	sc->retune_paused--;
+	return (err);
 }
 
 static int
@@ -649,7 +736,8 @@ mmc_sd_switch(struct mmc_softc *sc, uint8_t mode, uint8_t grp, uint8_t value,
 }
 
 static int
-mmc_set_card_bus_width(struct mmc_softc *sc, struct mmc_ivars *ivar)
+mmc_set_card_bus_width(struct mmc_softc *sc, struct mmc_ivars *ivar,
+    enum mmc_bus_timing timing)
 {
 	struct mmc_command cmd;
 	int err;
@@ -682,28 +770,33 @@ mmc_set_card_bus_width(struct mmc_softc *sc, struct mmc_ivars *ivar)
 	} else {
 		switch (ivar->bus_width) {
 		case bus_width_1:
+			if (timing == bus_timing_mmc_hs400 ||
+			    timing == bus_timing_mmc_hs400es)
+				return (MMC_ERR_INVALID);
 			value = EXT_CSD_BUS_WIDTH_1;
 			break;
 		case bus_width_4:
-			switch (mmcbr_get_timing(sc->dev)) {
+			switch (timing) {
 			case bus_timing_mmc_ddr52:
-			case bus_timing_mmc_hs200:
-			case bus_timing_mmc_hs400:
-			case bus_timing_mmc_hs400es:
 				value = EXT_CSD_BUS_WIDTH_4_DDR;
 				break;
+			case bus_timing_mmc_hs400:
+			case bus_timing_mmc_hs400es:
+				return (MMC_ERR_INVALID);
 			default:
 				value = EXT_CSD_BUS_WIDTH_4;
 				break;
 			}
 			break;
 		case bus_width_8:
-			switch (mmcbr_get_timing(sc->dev)) {
+			value = 0;
+			switch (timing) {
+			case bus_timing_mmc_hs400es:
+				value = EXT_CSD_BUS_WIDTH_ES;
+				/* FALLTHROUGH */
 			case bus_timing_mmc_ddr52:
-			case bus_timing_mmc_hs200:
 			case bus_timing_mmc_hs400:
-			case bus_timing_mmc_hs400es:
-				value = EXT_CSD_BUS_WIDTH_8_DDR;
+				value |= EXT_CSD_BUS_WIDTH_8_DDR;
 				break;
 			default:
 				value = EXT_CSD_BUS_WIDTH_8;
@@ -828,6 +921,13 @@ mmc_set_timing(struct mmc_softc *sc, struct mmc_ivars *ivar,
 		case bus_timing_mmc_ddr52:
 			value = EXT_CSD_HS_TIMING_HS;
 			break;
+		case bus_timing_mmc_hs200:
+			value = EXT_CSD_HS_TIMING_HS200;
+			break;
+		case bus_timing_mmc_hs400:
+		case bus_timing_mmc_hs400es:
+			value = EXT_CSD_HS_TIMING_HS400;
+			break;
 		default:
 			return (MMC_ERR_INVALID);
 		}
@@ -844,6 +944,23 @@ mmc_set_timing(struct mmc_softc *sc, struct mmc_ivars *ivar,
 	return (err);
 }
 
+static int
+mmc_set_vccq(struct mmc_softc *sc, struct mmc_ivars *ivar,
+    enum mmc_bus_timing timing)
+{
+
+	if (isset(&ivar->vccq_120, timing))
+		mmcbr_set_vccq(sc->dev, vccq_120);
+	else if (isset(&ivar->vccq_180, timing))
+		mmcbr_set_vccq(sc->dev, vccq_180);
+	else
+		mmcbr_set_vccq(sc->dev, vccq_330);
+	if (mmcbr_switch_vccq(sc->dev) != 0)
+		return (MMC_ERR_INVALID);
+	else
+		return (MMC_ERR_NONE);
+}
+
 static const uint8_t p8[8] = {
 	0x55, 0xAA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
 };
@@ -1051,7 +1168,7 @@ static const int cur_max[8] = {
 	1000, 5000, 10000, 25000, 35000, 45000, 800000, 200000
 };
 
-static void
+static int
 mmc_decode_csd_sd(uint32_t *raw_csd, struct mmc_csd *csd)
 {
 	int v;
@@ -1092,6 +1209,7 @@ mmc_decode_csd_sd(uint32_t *raw_csd, struct mmc_csd *csd)
 		csd->r2w_factor = 1 << mmc_get_bits(raw_csd, 128, 26, 3);
 		csd->write_bl_len = 1 << mmc_get_bits(raw_csd, 128, 22, 4);
 		csd->write_bl_partial = mmc_get_bits(raw_csd, 128, 21, 1);
+		return (MMC_ERR_NONE);
 	} else if (v == 1) {
 		m = mmc_get_bits(raw_csd, 128, 115, 4);
 		e = mmc_get_bits(raw_csd, 128, 112, 3);
@@ -1115,8 +1233,9 @@ mmc_decode_csd_sd(uint32_t *raw_csd, struct mmc_csd *csd)
 		csd->r2w_factor = 1 << mmc_get_bits(raw_csd, 128, 26, 3);
 		csd->write_bl_len = 1 << mmc_get_bits(raw_csd, 128, 22, 4);
 		csd->write_bl_partial = mmc_get_bits(raw_csd, 128, 21, 1);
-	} else
-		panic("unknown SD CSD version");
+		return (MMC_ERR_NONE);
+	}
+	return (MMC_ERR_INVALID);
 }
 
 static void
@@ -1380,6 +1499,53 @@ mmc_timing_to_string(enum mmc_bus_timing timing)
 	return ("");
 }
 
+static bool
+mmc_host_timing(device_t dev, enum mmc_bus_timing timing)
+{
+	int host_caps;
+
+	host_caps = mmcbr_get_caps(dev);
+
+#define	HOST_TIMING_CAP(host_caps, cap) ({				\
+	bool retval;							\
+	if (((host_caps) & (cap)) == (cap))				\
+		retval = true;						\
+	else								\
+		retval = false;						\
+	retval;								\
+})
+
+	switch (timing) {
+	case bus_timing_normal:
+		return (true);
+	case bus_timing_hs:
+		return (HOST_TIMING_CAP(host_caps, MMC_CAP_HSPEED));
+	case bus_timing_uhs_sdr12:
+		return (HOST_TIMING_CAP(host_caps, MMC_CAP_UHS_SDR12));
+	case bus_timing_uhs_sdr25:
+		return (HOST_TIMING_CAP(host_caps, MMC_CAP_UHS_SDR25));
+	case bus_timing_uhs_ddr50:
+		return (HOST_TIMING_CAP(host_caps, MMC_CAP_UHS_DDR50));
+	case bus_timing_uhs_sdr50:
+		return (HOST_TIMING_CAP(host_caps, MMC_CAP_UHS_SDR50));
+	case bus_timing_uhs_sdr104:
+		return (HOST_TIMING_CAP(host_caps, MMC_CAP_UHS_SDR104));
+	case bus_timing_mmc_ddr52:
+		return (HOST_TIMING_CAP(host_caps, MMC_CAP_MMC_DDR52));
+	case bus_timing_mmc_hs200:
+		return (HOST_TIMING_CAP(host_caps, MMC_CAP_MMC_HS200));
+	case bus_timing_mmc_hs400:
+		return (HOST_TIMING_CAP(host_caps, MMC_CAP_MMC_HS400));
+	case bus_timing_mmc_hs400es:
+		return (HOST_TIMING_CAP(host_caps, MMC_CAP_MMC_HS400 |
+		    MMC_CAP_MMC_ENH_STROBE));
+	}
+
+#undef HOST_TIMING_CAP
+
+	return (false);
+}
+
 static void
 mmc_log_card(device_t dev, struct mmc_ivars *ivar, int newcard)
 {
@@ -1411,9 +1577,8 @@ mmc_discover_cards(struct mmc_softc *sc)
 	u_char switch_res[64];
 	uint32_t raw_cid[4];
 	struct mmc_ivars *ivar = NULL;
-	device_t *devlist;
 	device_t child;
-	int devcount, err, host_caps, i, newcard;
+	int err, host_caps, i, newcard;
 	uint32_t resp, sec_count, status;
 	uint16_t rca = 2;
 
@@ -1421,6 +1586,7 @@ mmc_discover_cards(struct mmc_softc *sc)
 	if (bootverbose || mmc_debug)
 		device_printf(sc->dev, "Probing cards\n");
 	while (1) {
+		child = NULL;
 		sc->squelched++; /* Errors are expected, squelch reporting. */
 		err = mmc_all_send_cid(sc, raw_cid);
 		sc->squelched--;
@@ -1431,18 +1597,14 @@ mmc_discover_cards(struct mmc_softc *sc)
 			break;
 		}
 		newcard = 1;
-		if ((err = device_get_children(sc->dev, &devlist,
-		    &devcount)) != 0)
-			return;
-		for (i = 0; i < devcount; i++) {
-			ivar = device_get_ivars(devlist[i]);
+		for (i = 0; i < sc->child_count; i++) {
+			ivar = device_get_ivars(sc->child_list[i]);
 			if (memcmp(ivar->raw_cid, raw_cid, sizeof(raw_cid)) ==
 			    0) {
 				newcard = 0;
 				break;
 			}
 		}
-		free(devlist, M_TEMP);
 		if (bootverbose || mmc_debug) {
 			device_printf(sc->dev,
 			    "%sard detected (CID %08x%08x%08x%08x)\n",
@@ -1465,7 +1627,7 @@ mmc_discover_cards(struct mmc_softc *sc)
 			if (err != MMC_ERR_NONE) {
 				device_printf(sc->dev,
 				    "Error getting RCA %d\n", err);
-				break;
+				goto free_ivar;
 			}
 			ivar->rca = resp >> 16;
 			/* Get card CSD. */
@@ -1473,7 +1635,7 @@ mmc_discover_cards(struct mmc_softc *sc)
 			if (err != MMC_ERR_NONE) {
 				device_printf(sc->dev,
 				    "Error getting CSD %d\n", err);
-				break;
+				goto free_ivar;
 			}
 			if (bootverbose || mmc_debug)
 				device_printf(sc->dev,
@@ -1481,7 +1643,11 @@ mmc_discover_cards(struct mmc_softc *sc)
 				    newcard ? "New c" : "C", ivar->raw_csd[0],
 				    ivar->raw_csd[1], ivar->raw_csd[2],
 				    ivar->raw_csd[3]);
-			mmc_decode_csd_sd(ivar->raw_csd, &ivar->csd);
+			err = mmc_decode_csd_sd(ivar->raw_csd, &ivar->csd);
+			if (err != MMC_ERR_NONE) {
+				device_printf(sc->dev, "Error decoding CSD\n");
+				goto free_ivar;
+			}
 			ivar->sec_count = ivar->csd.capacity / MMC_SECTOR_SIZE;
 			if (ivar->csd.csd_structure > 0)
 				ivar->high_cap = 1;
@@ -1494,12 +1660,12 @@ mmc_discover_cards(struct mmc_softc *sc)
 			if (err != MMC_ERR_NONE) {
 				device_printf(sc->dev,
 				    "Error reading card status %d\n", err);
-				break;
+				goto free_ivar;
 			}
 			if ((status & R1_CARD_IS_LOCKED) != 0) {
 				device_printf(sc->dev,
-				    "Card is password protected, skipping.\n");
-				break;
+				    "Card is password protected, skipping\n");
+				goto free_ivar;
 			}
 
 			/* Get card SCR.  Card must be selected to fetch it. */
@@ -1507,13 +1673,13 @@ mmc_discover_cards(struct mmc_softc *sc)
 			if (err != MMC_ERR_NONE) {
 				device_printf(sc->dev,
 				    "Error selecting card %d\n", err);
-				break;
+				goto free_ivar;
 			}
 			err = mmc_app_send_scr(sc, ivar->rca, ivar->raw_scr);
 			if (err != MMC_ERR_NONE) {
 				device_printf(sc->dev,
 				    "Error reading SCR %d\n", err);
-				break;
+				goto free_ivar;
 			}
 			mmc_app_decode_scr(ivar->raw_scr, &ivar->scr);
 			/* Get card switch capabilities (command class 10). */
@@ -1541,7 +1707,7 @@ mmc_discover_cards(struct mmc_softc *sc)
 			 * use from the sd_status is the erase sector size, but
 			 * it is still nice to get that right.
 			 */
-			mmc_select_card(sc, 0);
+			(void)mmc_select_card(sc, 0);
 			(void)mmc_select_card(sc, ivar->rca);
 			(void)mmc_app_sd_status(sc, ivar->rca,
 			    ivar->raw_sd_status);
@@ -1551,47 +1717,24 @@ mmc_discover_cards(struct mmc_softc *sc)
 				ivar->erase_sector =
 				    16 << ivar->sd_status.au_size;
 			}
-			/* Find max supported bus width. */
+			/* Find maximum supported bus width. */
 			if ((host_caps & MMC_CAP_4_BIT_DATA) &&
 			    (ivar->scr.bus_widths & SD_SCR_BUS_WIDTH_4))
 				ivar->bus_width = bus_width_4;
 
-			/*
-			 * Some cards that report maximum I/O block sizes
-			 * greater than 512 require the block length to be
-			 * set to 512, even though that is supposed to be
-			 * the default.  Example:
-			 *
-			 * Transcend 2GB SDSC card, CID:
-			 * mid=0x1b oid=0x534d pnm="00000" prv=1.0 mdt=00.2000
-			 */
-			if (ivar->csd.read_bl_len != MMC_SECTOR_SIZE ||
-			    ivar->csd.write_bl_len != MMC_SECTOR_SIZE)
-				mmc_set_blocklen(sc, MMC_SECTOR_SIZE);
-
-			mmc_format_card_id_string(ivar);
-
-			if (bootverbose || mmc_debug)
-				mmc_log_card(sc->dev, ivar, newcard);
-			if (newcard) {
-				/* Add device. */
-				child = device_add_child(sc->dev, NULL, -1);
-				device_set_ivars(child, ivar);
-			}
-			mmc_select_card(sc, 0);
-			return;
+			goto child_common;
 		}
 		ivar->rca = rca++;
 		err = mmc_set_relative_addr(sc, ivar->rca);
 		if (err != MMC_ERR_NONE) {
 			device_printf(sc->dev, "Error setting RCA %d\n", err);
-			break;
+			goto free_ivar;
 		}
 		/* Get card CSD. */
 		err = mmc_send_csd(sc, ivar->rca, ivar->raw_csd);
 		if (err != MMC_ERR_NONE) {
 			device_printf(sc->dev, "Error getting CSD %d\n", err);
-			break;
+			goto free_ivar;
 		}
 		if (bootverbose || mmc_debug)
 			device_printf(sc->dev,
@@ -1610,19 +1753,19 @@ mmc_discover_cards(struct mmc_softc *sc)
 		if (err != MMC_ERR_NONE) {
 			device_printf(sc->dev,
 			    "Error reading card status %d\n", err);
-			break;
+			goto free_ivar;
 		}
 		if ((status & R1_CARD_IS_LOCKED) != 0) {
 			device_printf(sc->dev,
-			    "Card is password protected, skipping.\n");
-			break;
+			    "Card is password protected, skipping\n");
+			goto free_ivar;
 		}
 
 		err = mmc_select_card(sc, ivar->rca);
 		if (err != MMC_ERR_NONE) {
 			device_printf(sc->dev, "Error selecting card %d\n",
 			    err);
-			break;
+			goto free_ivar;
 		}
 
 		/* Only MMC >= 4.x devices support EXT_CSD. */
@@ -1632,7 +1775,7 @@ mmc_discover_cards(struct mmc_softc *sc)
 			if (err != MMC_ERR_NONE) {
 				device_printf(sc->dev,
 				    "Error reading EXT_CSD %d\n", err);
-				break;
+				goto free_ivar;
 			}
 			/* Handle extended capacity from EXT_CSD */
 			sec_count = ivar->raw_ext_csd[EXT_CSD_SEC_CNT] +
@@ -1643,6 +1786,8 @@ mmc_discover_cards(struct mmc_softc *sc)
 				ivar->sec_count = sec_count;
 				ivar->high_cap = 1;
 			}
+			/* Find maximum supported bus width. */
+			ivar->bus_width = mmc_test_bus_width(sc);
 			/* Get device speeds beyond normal mode. */
 			if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] &
 			    EXT_CSD_CARD_TYPE_HS_52) != 0) {
@@ -1665,6 +1810,50 @@ mmc_discover_cards(struct mmc_softc *sc)
 				setbit(&ivar->timings, bus_timing_mmc_ddr52);
 				setbit(&ivar->vccq_180, bus_timing_mmc_ddr52);
 			}
+			if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] &
+			    EXT_CSD_CARD_TYPE_HS200_1_2V) != 0 &&
+			    (host_caps & MMC_CAP_SIGNALING_120) != 0) {
+				setbit(&ivar->timings, bus_timing_mmc_hs200);
+				setbit(&ivar->vccq_120, bus_timing_mmc_hs200);
+			}
+			if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] &
+			    EXT_CSD_CARD_TYPE_HS200_1_8V) != 0 &&
+			    (host_caps & MMC_CAP_SIGNALING_180) != 0) {
+				setbit(&ivar->timings, bus_timing_mmc_hs200);
+				setbit(&ivar->vccq_180, bus_timing_mmc_hs200);
+			}
+			if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] &
+			    EXT_CSD_CARD_TYPE_HS400_1_2V) != 0 &&
+			    (host_caps & MMC_CAP_SIGNALING_120) != 0 &&
+			    ivar->bus_width == bus_width_8) {
+				setbit(&ivar->timings, bus_timing_mmc_hs400);
+				setbit(&ivar->vccq_120, bus_timing_mmc_hs400);
+			}
+			if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] &
+			    EXT_CSD_CARD_TYPE_HS400_1_8V) != 0 &&
+			    (host_caps & MMC_CAP_SIGNALING_180) != 0 &&
+			    ivar->bus_width == bus_width_8) {
+				setbit(&ivar->timings, bus_timing_mmc_hs400);
+				setbit(&ivar->vccq_180, bus_timing_mmc_hs400);
+			}
+			if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] &
+			    EXT_CSD_CARD_TYPE_HS400_1_2V) != 0 &&
+			    (ivar->raw_ext_csd[EXT_CSD_STROBE_SUPPORT] &
+			    EXT_CSD_STROBE_SUPPORT_EN) != 0 &&
+			    (host_caps & MMC_CAP_SIGNALING_120) != 0 &&
+			    ivar->bus_width == bus_width_8) {
+				setbit(&ivar->timings, bus_timing_mmc_hs400es);
+				setbit(&ivar->vccq_120, bus_timing_mmc_hs400es);
+			}
+			if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] &
+			    EXT_CSD_CARD_TYPE_HS400_1_8V) != 0 &&
+			    (ivar->raw_ext_csd[EXT_CSD_STROBE_SUPPORT] &
+			    EXT_CSD_STROBE_SUPPORT_EN) != 0 &&
+			    (host_caps & MMC_CAP_SIGNALING_180) != 0 &&
+			    ivar->bus_width == bus_width_8) {
+				setbit(&ivar->timings, bus_timing_mmc_hs400es);
+				setbit(&ivar->vccq_180, bus_timing_mmc_hs400es);
+			}
 			/*
 			 * Determine generic switch timeout (provided in
 			 * units of 10 ms), defaulting to 500 ms.
@@ -1673,8 +1862,6 @@ mmc_discover_cards(struct mmc_softc *sc)
 			if (ivar->csd.spec_vers >= 6)
 				ivar->cmd6_time = 10 *
 				    ivar->raw_ext_csd[EXT_CSD_GEN_CMD6_TIME];
-			/* Find max supported bus width. */
-			ivar->bus_width = mmc_test_bus_width(sc);
 			/* Handle HC erase sector size. */
 			if (ivar->raw_ext_csd[EXT_CSD_ERASE_GRP_SIZE] != 0) {
 				ivar->erase_sector = 1024 *
@@ -1688,11 +1875,15 @@ mmc_discover_cards(struct mmc_softc *sc)
 					device_printf(sc->dev,
 					    "Error setting erase group %d\n",
 					    err);
-					break;
+					goto free_ivar;
 				}
 			}
 		}
 
+		mmc_decode_cid_mmc(ivar->raw_cid, &ivar->cid,
+		    ivar->raw_ext_csd[EXT_CSD_REV] >= 5);
+
+child_common:
 		/*
 		 * Some cards that report maximum I/O block sizes greater
 		 * than 512 require the block length to be set to 512, even
@@ -1705,8 +1896,6 @@ mmc_discover_cards(struct mmc_softc *sc)
 		    ivar->csd.write_bl_len != MMC_SECTOR_SIZE)
 			mmc_set_blocklen(sc, MMC_SECTOR_SIZE);
 
-		mmc_decode_cid_mmc(ivar->raw_cid, &ivar->cid,
-		    ivar->raw_ext_csd[EXT_CSD_REV] >= 5);
 		mmc_format_card_id_string(ivar);
 
 		if (bootverbose || mmc_debug)
@@ -1714,56 +1903,111 @@ mmc_discover_cards(struct mmc_softc *sc)
 		if (newcard) {
 			/* Add device. */
 			child = device_add_child(sc->dev, NULL, -1);
-			device_set_ivars(child, ivar);
+			if (child != NULL) {
+				device_set_ivars(child, ivar);
+				sc->child_list = realloc(sc->child_list,
+				    sizeof(device_t) * sc->child_count + 1,
+				    M_DEVBUF, M_WAITOK);
+				sc->child_list[sc->child_count++] = child;
+			} else
+				device_printf(sc->dev, "Error adding child\n");
 		}
-		mmc_select_card(sc, 0);
+
+free_ivar:
+		if (newcard && child == NULL)
+			free(ivar, M_DEVBUF);
+		(void)mmc_select_card(sc, 0);
+		/*
+		 * Not returning here when one MMC device could no be added
+		 * potentially would mean looping forever when that device
+		 * is broken (in which case it also may impact the remainder
+		 * of the bus anyway, though).
+		 */
+		if ((newcard && child == NULL) ||
+		    mmcbr_get_mode(sc->dev) == mode_sd)
+			return;
 	}
 }
 
 static void
+mmc_update_child_list(struct mmc_softc *sc)
+{
+	device_t child;
+	int i, j;
+
+	if (sc->child_count == 0) {
+		free(sc->child_list, M_DEVBUF);
+		return;
+	}
+	for (i = j = 0; i < sc->child_count; i++) {
+		for (;;) {
+			child = sc->child_list[j++];
+			if (child != NULL)
+				break;
+		}
+		if (i != j)
+			sc->child_list[i] = child;
+	}
+	sc->child_list = realloc(sc->child_list, sizeof(device_t) *
+	    sc->child_count, M_DEVBUF, M_WAITOK);
+}
+
+static void
 mmc_rescan_cards(struct mmc_softc *sc)
 {
 	struct mmc_ivars *ivar;
-	device_t *devlist;
-	int err, i, devcount;
+	int err, i, j;
 
-	if ((err = device_get_children(sc->dev, &devlist, &devcount)) != 0)
-		return;
-	for (i = 0; i < devcount; i++) {
-		ivar = device_get_ivars(devlist[i]);
+	for (i = j = 0; i < sc->child_count; i++) {
+		ivar = device_get_ivars(sc->child_list[i]);
 		if (mmc_select_card(sc, ivar->rca) != MMC_ERR_NONE) {
 			if (bootverbose || mmc_debug)
 				device_printf(sc->dev,
-				    "Card at relative address %d lost.\n",
+				    "Card at relative address %d lost\n",
 				    ivar->rca);
-			device_delete_child(sc->dev, devlist[i]);
+			err = device_delete_child(sc->dev, sc->child_list[i]);
+			if (err != 0) {
+				j++;
+				continue;
+			}
 			free(ivar, M_DEVBUF);
-		}
+		} else
+			j++;
 	}
-	free(devlist, M_TEMP);
-	mmc_select_card(sc, 0);
+	if (sc->child_count == j)
+		goto out;
+	sc->child_count = j;
+	mmc_update_child_list(sc);
+out:
+	(void)mmc_select_card(sc, 0);
 }
 
 static int
-mmc_delete_cards(struct mmc_softc *sc)
+mmc_delete_cards(struct mmc_softc *sc, bool final)
 {
 	struct mmc_ivars *ivar;
-	device_t *devlist;
-	int err, i, devcount;
+	int err, i, j;
 
-	if ((err = device_get_children(sc->dev, &devlist, &devcount)) != 0)
-		return (err);
-	for (i = 0; i < devcount; i++) {
-		ivar = device_get_ivars(devlist[i]);
+	err = 0;
+	for (i = j = 0; i < sc->child_count; i++) {
+		ivar = device_get_ivars(sc->child_list[i]);
 		if (bootverbose || mmc_debug)
 			device_printf(sc->dev,
-			    "Card at relative address %d deleted.\n",
+			    "Card at relative address %d deleted\n",
 			    ivar->rca);
-		device_delete_child(sc->dev, devlist[i]);
+		err = device_delete_child(sc->dev, sc->child_list[i]);
+		if (err != 0) {
+			j++;
+			if (final == false)
+				continue;
+			else
+				break;
+		}
 		free(ivar, M_DEVBUF);
 	}
-	free(devlist, M_TEMP);
-	return (0);
+	sc->child_count = j;
+	mmc_update_child_list(sc);
+	return (err);
 }
 
 static void
@@ -1827,7 +2071,7 @@ mmc_go_discovery(struct mmc_softc *sc)
 		    mmcbr_get_ocr(dev));
 	if (mmcbr_get_ocr(dev) == 0) {
 		device_printf(sc->dev, "No compatible cards found on bus\n");
-		mmc_delete_cards(sc);
+		(void)mmc_delete_cards(sc, false);
 		mmc_power_down(sc);
 		return;
 	}
@@ -1851,31 +2095,27 @@ mmc_go_discovery(struct mmc_softc *sc)
 static int
 mmc_calculate_clock(struct mmc_softc *sc)
 {
-	device_t *kids;
+	device_t dev;
 	struct mmc_ivars *ivar;
-	int host_caps, i, nkid;
+	int i;
 	uint32_t dtr, max_dtr;
+	uint16_t rca;
 	enum mmc_bus_timing max_timing, timing;
-	bool changed;
+	bool changed, hs400;
 
-	max_dtr = mmcbr_get_f_max(sc->dev);
-	host_caps = mmcbr_get_caps(sc->dev);
-	if ((host_caps & MMC_CAP_MMC_DDR52) != 0)
-		max_timing = bus_timing_mmc_ddr52;
-	else if ((host_caps & MMC_CAP_HSPEED) != 0)
-		max_timing = bus_timing_hs;
-	else
-		max_timing = bus_timing_normal;
-	if (device_get_children(sc->dev, &kids, &nkid) != 0)
-		panic("can't get children");
+	dev = sc->dev;
+	max_dtr = mmcbr_get_f_max(dev);
+	max_timing = bus_timing_max;
 	do {
 		changed = false;
-		for (i = 0; i < nkid; i++) {
-			ivar = device_get_ivars(kids[i]);
-			if (isclr(&ivar->timings, max_timing)) {
-				for (timing = max_timing; timing >=
+		for (i = 0; i < sc->child_count; i++) {
+			ivar = device_get_ivars(sc->child_list[i]);
+			if (isclr(&ivar->timings, max_timing) ||
+			    !mmc_host_timing(dev, max_timing)) {
+				for (timing = max_timing - 1; timing >=
 				    bus_timing_normal; timing--) {
-					if (isset(&ivar->timings, timing)) {
+					if (isset(&ivar->timings, timing) &&
+					    mmc_host_timing(dev, timing)) {
 						max_timing = timing;
 						break;
 					}
@@ -1889,38 +2129,316 @@ mmc_calculate_clock(struct mmc_softc *sc)
 			}
 		}
 	} while (changed == true);
+
 	if (bootverbose || mmc_debug) {
-		device_printf(sc->dev,
+		device_printf(dev,
 		    "setting transfer rate to %d.%03dMHz (%s timing)\n",
 		    max_dtr / 1000000, (max_dtr / 1000) % 1000,
 		    mmc_timing_to_string(max_timing));
 	}
-	for (i = 0; i < nkid; i++) {
-		ivar = device_get_ivars(kids[i]);
+
+	/*
+	 * HS400 must be tuned in HS200 mode, so in case of HS400 we begin
+	 * with HS200 following the sequence as described in "6.6.2.2 HS200
+	 * timing mode selection" of the eMMC specification v5.1, too, and
+	 * switch to max_timing later.  HS400ES requires no tuning and, thus,
+	 * can be switch to directly, but requires the same detour via high
+	 * speed mode as does HS400 (see mmc_switch_to_hs400()).
+	 */
+	hs400 = max_timing == bus_timing_mmc_hs400;
+	timing = hs400 == true ? bus_timing_mmc_hs200 : max_timing;
+	for (i = 0; i < sc->child_count; i++) {
+		ivar = device_get_ivars(sc->child_list[i]);
 		if ((ivar->timings & ~(1 << bus_timing_normal)) == 0)
 			continue;
-		if (mmc_select_card(sc, ivar->rca) != MMC_ERR_NONE ||
-		    mmc_set_timing(sc, ivar, max_timing) != MMC_ERR_NONE)
-			device_printf(sc->dev, "Card at relative address %d "
-			    "failed to set timing.\n", ivar->rca);
+
+		rca = ivar->rca;
+		if (mmc_select_card(sc, rca) != MMC_ERR_NONE) {
+			device_printf(dev, "Card at relative address %d "
+			    "failed to select\n", rca);
+			continue;
+		}
+
+		if (timing == bus_timing_mmc_hs200 ||	/* includes HS400 */
+		    timing == bus_timing_mmc_hs400es) {
+			if (mmc_set_vccq(sc, ivar, timing) != MMC_ERR_NONE) {
+				device_printf(dev, "Failed to set VCCQ for "
+				    "card at relative address %d\n", rca);
+				continue;
+			}
+		}
+
+		if (timing == bus_timing_mmc_hs200) {	/* includes HS400 */
+			/* Set bus width (required for initial tuning). */
+			if (mmc_set_card_bus_width(sc, ivar, timing) !=
+			    MMC_ERR_NONE) {
+				device_printf(dev, "Card at relative address "
+				    "%d failed to set bus width\n", rca);
+				continue;
+			}
+			mmcbr_set_bus_width(dev, ivar->bus_width);
+			mmcbr_update_ios(dev);
+		} else if (timing == bus_timing_mmc_hs400es) {
+			if (mmc_switch_to_hs400(sc, ivar, max_dtr, timing) !=
+			    MMC_ERR_NONE) {
+				device_printf(dev, "Card at relative address "
+				    "%d failed to set %s timing\n", rca,
+				    mmc_timing_to_string(timing));
+				continue;
+			}
+			goto power_class;
+		}
+
+		if (mmc_set_timing(sc, ivar, timing) != MMC_ERR_NONE) {
+			device_printf(dev, "Card at relative address %d "
+			    "failed to set %s timing\n", rca,
+			    mmc_timing_to_string(timing));
+			continue;
+		}
+
+		if (timing == bus_timing_mmc_ddr52) {
+			/*
+			 * Set EXT_CSD_BUS_WIDTH_n_DDR in EXT_CSD_BUS_WIDTH
+			 * (must be done after switching to EXT_CSD_HS_TIMING).
+			 */
+			if (mmc_set_card_bus_width(sc, ivar, timing) !=
+			    MMC_ERR_NONE) {
+				device_printf(dev, "Card at relative address "
+				    "%d failed to set bus width\n", rca);
+				continue;
+			}
+			mmcbr_set_bus_width(dev, ivar->bus_width);
+			mmcbr_update_ios(dev);
+			if (mmc_set_vccq(sc, ivar, timing) != MMC_ERR_NONE) {
+				device_printf(dev, "Failed to set VCCQ for "
+				    "card at relative address %d\n", rca);
+				continue;
+			}
+		}
+
+		/* Set clock (must be done before initial tuning). */
+		mmcbr_set_clock(dev, max_dtr);
+		mmcbr_update_ios(dev);
+
+		if (mmcbr_tune(dev, hs400) != 0) {
+			device_printf(dev, "Card at relative address %d "
+			    "failed to execute initial tuning\n", rca);
+			continue;
+		}
+
+		if (hs400 == true && mmc_switch_to_hs400(sc, ivar, max_dtr,
+		    max_timing) != MMC_ERR_NONE) {
+			device_printf(dev, "Card at relative address %d "
+			    "failed to set %s timing\n", rca,
+			    mmc_timing_to_string(max_timing));
+			continue;
+		}
+
+power_class:
+		if (mmc_set_power_class(sc, ivar) != MMC_ERR_NONE) {
+			device_printf(dev, "Card at relative address %d "
+			    "failed to set power class\n", rca);
+		}
 	}
-	mmc_select_card(sc, 0);
-	free(kids, M_TEMP);
-	mmcbr_set_clock(sc->dev, max_dtr);
-	mmcbr_update_ios(sc->dev);
+	(void)mmc_select_card(sc, 0);
 	return (max_dtr);
 }
 
+/*
+ * Switch from HS200 to HS400 (either initially or for re-tuning) or directly
+ * to HS400ES.  This follows the sequences described in "6.6.2.3 HS400 timing
+ * mode selection" of the eMMC specification v5.1.
+ */
+static int
+mmc_switch_to_hs400(struct mmc_softc *sc, struct mmc_ivars *ivar,
+    uint32_t clock, enum mmc_bus_timing max_timing)
+{
+	device_t dev;
+	int err;
+	uint16_t rca;
+
+	dev = sc->dev;
+	rca = ivar->rca;
+
+	/*
+	 * Both clock and timing must be set as appropriate for high speed
+	 * before eventually switching to HS400/HS400ES; mmc_set_timing()
+	 * will issue mmcbr_update_ios().
+	 */
+	mmcbr_set_clock(dev, ivar->hs_tran_speed);
+	err = mmc_set_timing(sc, ivar, bus_timing_hs);
+	if (err != MMC_ERR_NONE)
+		return (err);
+
+	/*
+	 * Set EXT_CSD_BUS_WIDTH_8_DDR in EXT_CSD_BUS_WIDTH (and additionally
+	 * EXT_CSD_BUS_WIDTH_ES for HS400ES).
+	 */
+	err = mmc_set_card_bus_width(sc, ivar, max_timing);
+	if (err != MMC_ERR_NONE)
+		return (err);
+	mmcbr_set_bus_width(dev, ivar->bus_width);
+	mmcbr_update_ios(dev);
+
+	/* Finally, switch to HS400/HS400ES mode. */
+	err = mmc_set_timing(sc, ivar, max_timing);
+	if (err != MMC_ERR_NONE)
+		return (err);
+	mmcbr_set_clock(dev, clock);
+	mmcbr_update_ios(dev);
+	return (MMC_ERR_NONE);
+}
+
+/*
+ * Switch from HS400 to HS200 (for re-tuning).
+ */
+static int
+mmc_switch_to_hs200(struct mmc_softc *sc, struct mmc_ivars *ivar,
+    uint32_t clock)
+{
+	device_t dev;
+	int err;
+	uint16_t rca;
+
+	dev = sc->dev;
+	rca = ivar->rca;
+
+	/*
+	 * Both clock and timing must initially be set as appropriate for
+	 * DDR52 before eventually switching to HS200; mmc_set_timing()
+	 * will issue mmcbr_update_ios().
+	 */
+	mmcbr_set_clock(dev, ivar->hs_tran_speed);
+	err = mmc_set_timing(sc, ivar, bus_timing_mmc_ddr52);
+	if (err != MMC_ERR_NONE)
+		return (err);
+
+	/*
+	 * Next, switch to high speed.  Thus, clear EXT_CSD_BUS_WIDTH_n_DDR
+	 * in EXT_CSD_BUS_WIDTH and update bus width and timing in ios.
+	 */
+	err = mmc_set_card_bus_width(sc, ivar, bus_timing_hs);
+	if (err != MMC_ERR_NONE)
+		return (err);
+	mmcbr_set_bus_width(dev, ivar->bus_width);
+	mmcbr_set_timing(sc->dev, bus_timing_hs);
+	mmcbr_update_ios(dev);
+
+	/* Finally, switch to HS200 mode. */
+	err = mmc_set_timing(sc, ivar, bus_timing_mmc_hs200);
+	if (err != MMC_ERR_NONE)
+		return (err);
+	mmcbr_set_clock(dev, clock);
+	mmcbr_update_ios(dev);
+	return (MMC_ERR_NONE);
+}
+
+static int
+mmc_retune(device_t busdev, device_t dev, bool reset)
+{
+	struct mmc_softc *sc;
+	struct mmc_ivars *ivar;
+	int err;
+	uint32_t clock;
+	enum mmc_bus_timing timing;
+
+	if (device_get_parent(dev) != busdev)
+		return (MMC_ERR_INVALID);
+
+	sc = device_get_softc(busdev);
+	if (sc->retune_needed != 1 && sc->retune_paused != 0)
+		return (MMC_ERR_INVALID);
+
+	timing = mmcbr_get_timing(busdev);
+	if (timing == bus_timing_mmc_hs400) {
+		/*
+		 * Controllers use the data strobe line to latch data from
+		 * the devices in HS400 mode so periodic re-tuning isn't
+		 * expected to be required, i. e. only if a CRC or tuning
+		 * error is signaled to the bridge.  In these latter cases
+		 * we are asked to reset the tuning circuit and need to do
+		 * the switch timing dance.
+		 */
+		if (reset == false)
+			return (0);
+		ivar = device_get_ivars(dev);
+		clock = mmcbr_get_clock(busdev);
+		if (mmc_switch_to_hs200(sc, ivar, clock) != MMC_ERR_NONE)
+			return (MMC_ERR_BADCRC);
+	}
+	err = mmcbr_retune(busdev, reset);
+	if (err != 0 && timing == bus_timing_mmc_hs400)
+		return (MMC_ERR_BADCRC);
+	switch (err) {
+	case 0:
+		break;
+	case EIO:
+		return (MMC_ERR_FAILED);
+	default:
+		return (MMC_ERR_INVALID);
+	}
+	if (timing == bus_timing_mmc_hs400) {
+		if (mmc_switch_to_hs400(sc, ivar, clock, timing) !=
+		    MMC_ERR_NONE)
+			return (MMC_ERR_BADCRC);
+	}
+	return (MMC_ERR_NONE);
+}
+
+static void
+mmc_retune_pause(device_t busdev, device_t dev, bool retune)
+{
+	struct mmc_softc *sc;
+
+	sc = device_get_softc(busdev);
+	KASSERT(device_get_parent(dev) == busdev,
+	    ("%s: %s is not a child of %s", __func__, device_get_nameunit(dev),
+	    device_get_nameunit(busdev)));
+	KASSERT(sc->owner != NULL,
+	    ("%s: Request from %s without bus being acquired.", __func__,
+	    device_get_nameunit(dev)));
+
+	if (retune == true && sc->retune_paused == 0)
+		sc->retune_needed = 1;
+	sc->retune_paused++;
+}
+
+static void
+mmc_retune_unpause(device_t busdev, device_t dev)
+{
+	struct mmc_softc *sc;
+
+	sc = device_get_softc(busdev);
+	KASSERT(device_get_parent(dev) == busdev,
+	    ("%s: %s is not a child of %s", __func__, device_get_nameunit(dev),
+	    device_get_nameunit(busdev)));
+	KASSERT(sc->owner != NULL,
+	    ("%s: Request from %s without bus being acquired.", __func__,
+	    device_get_nameunit(dev)));
+	KASSERT(sc->retune_paused != 0,
+	    ("%s: Re-tune pause count already at 0", __func__));
+
+	sc->retune_paused--;
+}
+
 static void
 mmc_scan(struct mmc_softc *sc)
 {
 	device_t dev = sc->dev;
+	int err;
 
-	mmc_acquire_bus(dev, dev);
+	err = mmc_acquire_bus(dev, dev);
+	if (err != 0) {
+		device_printf(dev, "Failed to acquire bus for scanning\n");
+		return;
+	}
 	mmc_go_discovery(sc);
-	mmc_release_bus(dev, dev);
-
-	bus_generic_attach(dev);
+	err = mmc_release_bus(dev, dev);
+	if (err != 0) {
+		device_printf(dev, "Failed to release bus after scanning\n");
+		return;
+	}
+	(void)bus_generic_attach(dev);
 }
 
 static int
@@ -2019,6 +2537,8 @@ static device_method_t mmc_methods[] = {
 	DEVMETHOD(bus_child_location_str, mmc_child_location_str),
 
 	/* MMC Bus interface */
+	DEVMETHOD(mmcbus_retune_pause, mmc_retune_pause),
+	DEVMETHOD(mmcbus_retune_unpause, mmc_retune_unpause),
 	DEVMETHOD(mmcbus_wait_for_request, mmc_wait_for_request),
 	DEVMETHOD(mmcbus_acquire_bus, mmc_acquire_bus),
 	DEVMETHOD(mmcbus_release_bus, mmc_release_bus),
diff --git a/freebsd/sys/dev/mmc/mmc_ioctl.h b/freebsd/sys/dev/mmc/mmc_ioctl.h
index 97cff068..e633fec9 100644
--- a/freebsd/sys/dev/mmc/mmc_ioctl.h
+++ b/freebsd/sys/dev/mmc/mmc_ioctl.h
@@ -54,7 +54,7 @@ struct mmc_ioc_multi_cmd {
 #define	MMC_IOC_BASE		'M'
 
 #define	MMC_IOC_CMD		_IOWR(MMC_IOC_BASE, 0, struct mmc_ioc_cmd)
-#define	MMC_IOC_CMD_MULTI	_IOWR(MMC_IOC_BASE, 1, struct mmc_ioc_multi_cmd)
+#define	MMC_IOC_MULTI_CMD	_IOWR(MMC_IOC_BASE, 1, struct mmc_ioc_multi_cmd)
 
 /* Maximum accepted data transfer size */
 #define	MMC_IOC_MAX_BYTES	(512  * 256)
diff --git a/freebsd/sys/dev/mmc/mmc_private.h b/freebsd/sys/dev/mmc/mmc_private.h
index bbca0c60..633d0784 100644
--- a/freebsd/sys/dev/mmc/mmc_private.h
+++ b/freebsd/sys/dev/mmc/mmc_private.h
@@ -60,9 +60,14 @@ struct mmc_softc {
 	struct mtx sc_mtx;
 	struct intr_config_hook config_intrhook;
 	device_t owner;
-	uint32_t last_rca;
-	int	 squelched; /* suppress reporting of (expected) errors */
-	int	 log_count;
+	device_t *child_list;
+	int child_count;
+	uint16_t last_rca;
+	uint16_t retune_paused;
+	uint8_t retune_needed;
+	uint8_t retune_ongoing;
+	uint16_t squelched;	/* suppress reporting of (expected) errors */
+	int log_count;
 	struct timeval log_time;
 };
 
diff --git a/freebsd/sys/dev/mmc/mmc_subr.c b/freebsd/sys/dev/mmc/mmc_subr.c
index f76e9637..006354ba 100644
--- a/freebsd/sys/dev/mmc/mmc_subr.c
+++ b/freebsd/sys/dev/mmc/mmc_subr.c
@@ -140,7 +140,6 @@ mmc_wait_for_app_cmd(device_t brdev, device_t reqdev, uint16_t rca,
 	sc->squelched--;
 
 	if (err != MMC_ERR_NONE && brdev == reqdev) {
-		sc = device_get_softc(brdev);
 		if (sc->squelched == 0 && ppsratecheck(&sc->log_time,
 		    &sc->log_count, LOG_PPS)) {
 			device_printf(sc->dev, "ACMD%d failed, RESULT: %d\n",
@@ -156,10 +155,13 @@ mmc_switch(device_t brdev, device_t reqdev, uint16_t rca, uint8_t set,
     uint8_t index, uint8_t value, u_int timeout, bool status)
 {
 	struct mmc_command cmd;
+	struct mmc_softc *sc;
 	int err;
 
 	KASSERT(timeout != 0, ("%s: no timeout", __func__));
 
+	sc = device_get_softc(brdev);
+
 	memset(&cmd, 0, sizeof(cmd));
 	cmd.opcode = MMC_SWITCH_FUNC;
 	cmd.arg = (MMC_SWITCH_FUNC_WR << 24) | (index << 16) | (value << 8) |
@@ -174,10 +176,19 @@ mmc_switch(device_t brdev, device_t reqdev, uint16_t rca, uint8_t set,
 		cmd.flags = MMC_RSP_R1 | MMC_CMD_AC;
 	else
 		cmd.flags = MMC_RSP_R1B | MMC_CMD_AC;
+	/*
+	 * Pause re-tuning so it won't interfere with the busy state and also
+	 * so that the result of CMD13 will always refer to switching rather
+	 * than to a tuning command that may have snuck in between.
+	 */
+	sc->retune_paused++;
 	err = mmc_wait_for_cmd(brdev, reqdev, &cmd, CMD_RETRIES);
 	if (err != MMC_ERR_NONE || status == false)
-		return (err);
-	return (mmc_switch_status(brdev, reqdev, rca, timeout));
+		goto out;
+	err = mmc_switch_status(brdev, reqdev, rca, timeout);
+out:
+	sc->retune_paused--;
+	return (err);
 }
 
 int
@@ -194,6 +205,7 @@ mmc_switch_status(device_t brdev, device_t reqdev, uint16_t rca, u_int timeout)
 	 * type MMC_CAP_WAIT_WHILE_BUSY will issue mmc_send_status() only
 	 * once and then exit the loop.
 	 */
+	end.tv_sec = end.tv_usec = 0;
 	for (;;) {
 		err = mmc_send_status(brdev, reqdev, rca, &status);
 		if (err != MMC_ERR_NONE)
@@ -210,7 +222,7 @@ mmc_switch_status(device_t brdev, device_t reqdev, uint16_t rca, u_int timeout)
 			break;
 		}
 	}
-	if (err == MMC_ERR_NONE && R1_CURRENT_STATE(status) == R1_SWITCH_ERROR)
+	if (err == MMC_ERR_NONE && (status & R1_SWITCH_ERROR) != 0)
 		return (MMC_ERR_FAILED);
 	return (err);
 }
diff --git a/freebsd/sys/dev/mmc/mmcbrvar.h b/freebsd/sys/dev/mmc/mmcbrvar.h
index 77c304b4..c70af92a 100644
--- a/freebsd/sys/dev/mmc/mmcbrvar.h
+++ b/freebsd/sys/dev/mmc/mmcbrvar.h
@@ -70,6 +70,7 @@ enum mmcbr_device_ivars {
     MMCBR_IVAR_MODE,
     MMCBR_IVAR_OCR,
     MMCBR_IVAR_POWER_MODE,
+    MMCBR_IVAR_RETUNE_REQ,
     MMCBR_IVAR_VDD,
     MMCBR_IVAR_VCCQ,
     MMCBR_IVAR_CAPS,
@@ -94,6 +95,7 @@ MMCBR_ACCESSOR(host_ocr, HOST_OCR, int)
 MMCBR_ACCESSOR(mode, MODE, int)
 MMCBR_ACCESSOR(ocr, OCR, int)
 MMCBR_ACCESSOR(power_mode, POWER_MODE, int)
+MMCBR_ACCESSOR(retune_req, RETUNE_REQ, int)
 MMCBR_ACCESSOR(vdd, VDD, int)
 MMCBR_ACCESSOR(vccq, VCCQ, int)
 MMCBR_ACCESSOR(caps, CAPS, int)
@@ -109,6 +111,20 @@ mmcbr_update_ios(device_t dev)
 }
 
 static int __inline
+mmcbr_tune(device_t dev, bool hs400)
+{
+
+	return (MMCBR_TUNE(device_get_parent(dev), dev, hs400));
+}
+
+static int __inline
+mmcbr_retune(device_t dev, bool reset)
+{
+
+	return (MMCBR_RETUNE(device_get_parent(dev), dev, reset));
+}
+
+static int __inline
 mmcbr_switch_vccq(device_t dev)
 {
 
diff --git a/freebsd/sys/dev/mmc/mmcreg.h b/freebsd/sys/dev/mmc/mmcreg.h
index 39680ad6..80f433c1 100644
--- a/freebsd/sys/dev/mmc/mmcreg.h
+++ b/freebsd/sys/dev/mmc/mmcreg.h
@@ -1,6 +1,7 @@
 /*-
  * Copyright (c) 2006 M. Warner Losh.  All rights reserved.
  * Copyright (c) 2017 Marius Strobl <marius@FreeBSD.org>
+ * Copyright (c) 2015-2016 Ilya Bakulin <kibab@FreeBSD.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -159,6 +160,34 @@ struct mmc_command {
 #define	R1_STATE_PRG	7
 #define	R1_STATE_DIS	8
 
+/* R4 response (SDIO) */
+#define R4_IO_NUM_FUNCTIONS(ocr)	(((ocr) >> 28) & 0x3)
+#define R4_IO_MEM_PRESENT		(0x1<<27)
+#define R4_IO_OCR_MASK			0x00fffff0
+
+/*
+ * R5 responses
+ *
+ * Types (per SD 2.0 standard)
+ *e : error bit
+ *s : status bit
+ *r : detected and set for the actual command response
+ *x : Detected and set during command execution.  The host can get
+ *    the status by issuing a command with R1 response.
+ *
+ * Clear Condition (per SD 2.0 standard)
+ *a : according to the card current state.
+ *b : always related to the previous command.  reception of a valid
+ *    command will clear it (with a delay of one command).
+ *c : clear by read
+ */
+#define R5_COM_CRC_ERROR		(1u << 15)/* er, b */
+#define R5_ILLEGAL_COMMAND		(1u << 14)/* er, b */
+#define R5_IO_CURRENT_STATE_MASK	(3u << 12)/* s, b */
+#define R5_IO_CURRENT_STATE(x) 		(((x) & R5_IO_CURRENT_STATE_MASK) >> 12)
+#define R5_ERROR			(1u << 11)/* erx, c */
+#define R5_FUNCTION_NUMBER		(1u << 9)/* er, c */
+#define R5_OUT_OF_RANGE			(1u << 8)/* er, c */
 struct mmc_data {
 	size_t len;		/* size of the data */
 	size_t xfer_len;
@@ -176,10 +205,12 @@ struct mmc_request {
 	struct mmc_command *stop;
 	void (*done)(struct mmc_request *); /* Completion function */
 	void *done_data;		/* requestor set data */
-#ifndef __rtems__
 	uint32_t flags;
+#ifndef __rtems__
 #define	MMC_REQ_DONE	1
-#else /* __rtems__ */
+#endif /* __rtems__ */
+#define	MMC_TUNE_DONE	2
+#ifdef __rtems__
 	rtems_binary_semaphore req_done;
 #endif /* __rtems__ */
 };
@@ -194,6 +225,7 @@ struct mmc_request {
 #define	SD_SEND_RELATIVE_ADDR	3
 #define	MMC_SET_DSR		4
 #define	MMC_SLEEP_AWAKE		5
+#define IO_SEND_OP_COND		5
 #define	MMC_SWITCH_FUNC		6
 #define	 MMC_SWITCH_FUNC_CMDS	 0
 #define	 MMC_SWITCH_FUNC_SET	 1
@@ -276,7 +308,31 @@ struct mmc_request {
 
 /* Class 9: I/O cards (sd) */
 #define	SD_IO_RW_DIRECT		52
+/* CMD52 arguments */
+#define  SD_ARG_CMD52_READ		(0<<31)
+#define  SD_ARG_CMD52_WRITE		(1<<31)
+#define  SD_ARG_CMD52_FUNC_SHIFT		28
+#define  SD_ARG_CMD52_FUNC_MASK		0x7
+#define  SD_ARG_CMD52_EXCHANGE		(1<<27)
+#define  SD_ARG_CMD52_REG_SHIFT		9
+#define  SD_ARG_CMD52_REG_MASK		0x1ffff
+#define  SD_ARG_CMD52_DATA_SHIFT		0
+#define  SD_ARG_CMD52_DATA_MASK		0xff
+#define  SD_R5_DATA(resp)		((resp)[0] & 0xff)
+
 #define	SD_IO_RW_EXTENDED	53
+/* CMD53 arguments */
+#define  SD_ARG_CMD53_READ		(0<<31)
+#define  SD_ARG_CMD53_WRITE		(1<<31)
+#define  SD_ARG_CMD53_FUNC_SHIFT		28
+#define  SD_ARG_CMD53_FUNC_MASK		0x7
+#define  SD_ARG_CMD53_BLOCK_MODE		(1<<27)
+#define  SD_ARG_CMD53_INCREMENT		(1<<26)
+#define  SD_ARG_CMD53_REG_SHIFT		9
+#define  SD_ARG_CMD53_REG_MASK		0x1ffff
+#define  SD_ARG_CMD53_LENGTH_SHIFT	0
+#define  SD_ARG_CMD53_LENGTH_MASK	0x1ff
+#define  SD_ARG_CMD53_LENGTH_MAX		64 /* XXX should be 511? */
 
 /* Class 10: Switch function commands */
 #define	SD_SWITCH_FUNC		6
@@ -384,8 +440,8 @@ struct mmc_request {
 
 #define	EXT_CSD_HS_TIMING_BC		0
 #define	EXT_CSD_HS_TIMING_HS		1
-#define	EXT_CSD_HS_TIMING_DDR200	2
-#define	EXT_CSD_HS_TIMING_DDR400	3
+#define	EXT_CSD_HS_TIMING_HS200		2
+#define	EXT_CSD_HS_TIMING_HS400		3
 #define	EXT_CSD_HS_TIMING_DRV_STR_SHIFT	4
 
 #define	EXT_CSD_POWER_CLASS_8BIT_MASK	0xf0
@@ -401,7 +457,6 @@ struct mmc_request {
 #define	EXT_CSD_CARD_TYPE_HS200_1_2V	0x0020
 #define	EXT_CSD_CARD_TYPE_HS400_1_8V	0x0040
 #define	EXT_CSD_CARD_TYPE_HS400_1_2V	0x0080
-#define	EXT_CSD_CARD_TYPE_HS400ES	0x0100
 
 #define	EXT_CSD_BUS_WIDTH_1	0
 #define	EXT_CSD_BUS_WIDTH_4	1
@@ -410,6 +465,8 @@ struct mmc_request {
 #define	EXT_CSD_BUS_WIDTH_8_DDR	6
 #define	EXT_CSD_BUS_WIDTH_ES	0x80
 
+#define	EXT_CSD_STROBE_SUPPORT_EN	0x01
+
 #define	MMC_TYPE_HS_26_MAX		26000000
 #define	MMC_TYPE_HS_52_MAX		52000000
 #define	MMC_TYPE_DDR52_MAX		52000000
@@ -447,6 +504,54 @@ struct mmc_request {
 /* Specifications require 400 kHz max. during ID phase. */
 #define	SD_MMC_CARD_ID_FREQUENCY	400000
 
+/*
+ * SDIO Direct & Extended I/O
+ */
+#define SD_IO_RW_WR		(1u << 31)
+#define SD_IO_RW_FUNC(x)	(((x) & 0x7) << 28)
+#define SD_IO_RW_RAW		(1u << 27)
+#define SD_IO_RW_INCR		(1u << 26)
+#define SD_IO_RW_ADR(x)		(((x) & 0x1FFFF) << 9)
+#define SD_IO_RW_DAT(x)		(((x) & 0xFF) << 0)
+#define SD_IO_RW_LEN(x)		(((x) & 0xFF) << 0)
+
+#define SD_IOE_RW_LEN(x)	(((x) & 0x1FF) << 0)
+#define SD_IOE_RW_BLK		(1u << 27)
+
+/* Card Common Control Registers (CCCR) */
+#define SD_IO_CCCR_START		0x00000
+#define SD_IO_CCCR_SIZE			0x100
+#define SD_IO_CCCR_FN_ENABLE		0x02
+#define SD_IO_CCCR_FN_READY		0x03
+#define SD_IO_CCCR_INT_ENABLE		0x04
+#define SD_IO_CCCR_INT_PENDING		0x05
+#define SD_IO_CCCR_CTL			0x06
+#define  CCCR_CTL_RES			(1<<3)
+#define SD_IO_CCCR_BUS_WIDTH		0x07
+#define  CCCR_BUS_WIDTH_4		(1<<1)
+#define  CCCR_BUS_WIDTH_1		(1<<0)
+#define SD_IO_CCCR_CARDCAP		0x08
+#define SD_IO_CCCR_CISPTR		0x09 /* XXX 9-10, 10-11, or 9-12 */
+
+/* Function Basic Registers (FBR) */
+#define SD_IO_FBR_START			0x00100
+#define SD_IO_FBR_SIZE			0x00700
+
+/* Card Information Structure (CIS) */
+#define SD_IO_CIS_START			0x01000
+#define SD_IO_CIS_SIZE			0x17000
+
+/* CIS tuple codes (based on PC Card 16) */
+#define SD_IO_CISTPL_VERS_1		0x15
+#define SD_IO_CISTPL_MANFID		0x20
+#define SD_IO_CISTPL_FUNCID		0x21
+#define SD_IO_CISTPL_FUNCE		0x22
+#define SD_IO_CISTPL_END		0xff
+
+/* CISTPL_FUNCID codes */
+/* OpenBSD incorrectly defines 0x0c as FUNCTION_WLAN */
+/* #define SDMMC_FUNCTION_WLAN		0x0c */
+
 /* OCR bits */
 
 /*
@@ -562,6 +667,10 @@ struct mmc_sd_status
 #define	MMC_PART_GP_MAX	4
 #define	MMC_PART_MAX	8
 
+#define	MMC_TUNING_MAX		64	/* Maximum tuning iterations */
+#define	MMC_TUNING_LEN		64	/* Size of tuning data */
+#define	MMC_TUNING_LEN_HS200	128	/* Size of tuning data in HS200 mode */
+
 /*
  * Older versions of the MMC standard had a variable sector size.  However,
  * I've been able to find no old MMC or SD cards that have a non 512
diff --git a/freebsd/sys/dev/mmc/mmcsd.c b/freebsd/sys/dev/mmc/mmcsd.c
index 5066e250..195feae2 100644
--- a/freebsd/sys/dev/mmc/mmcsd.c
+++ b/freebsd/sys/dev/mmc/mmcsd.c
@@ -108,7 +108,8 @@ __FBSDID("$FreeBSD$");
 struct mmcsd_softc;
 
 struct mmcsd_part {
-	struct mtx part_mtx;
+	struct mtx disk_mtx;
+	struct mtx ioctl_mtx;
 	struct mmcsd_softc *sc;
 #ifndef __rtems__
 	struct disk *disk;
@@ -120,6 +121,7 @@ struct mmcsd_part {
 	u_int type;
 	int running;
 	int suspend;
+	int ioctl;
 	bool ro;
 	char name[MMCSD_PART_NAMELEN];
 };
@@ -129,6 +131,9 @@ struct mmcsd_softc {
 	device_t mmcbr;
 	struct mmcsd_part *part[MMC_PART_MAX];
 	enum mmc_card_mode mode;
+	u_int max_data;		/* Maximum data size [blocks] */
+	u_int erase_sector;	/* Device native erase sector size [blocks] */
+	uint8_t	high_cap;	/* High Capacity device (block addressed) */
 	uint8_t part_curr;	/* Partition currently switched to */
 	uint8_t ext_csd[MMC_EXTCSD_SIZE];
 	uint16_t rca;
@@ -199,15 +204,25 @@ static int mmcsd_slicer(device_t dev, const char *provider,
 static int mmcsd_switch_part(device_t bus, device_t dev, uint16_t rca,
     u_int part);
 
-#define	MMCSD_PART_LOCK(_part)		mtx_lock(&(_part)->part_mtx)
-#define	MMCSD_PART_UNLOCK(_part)	mtx_unlock(&(_part)->part_mtx)
-#define	MMCSD_PART_LOCK_INIT(_part)					\
-	mtx_init(&(_part)->part_mtx, (_part)->name, "mmcsd part", MTX_DEF)
-#define	MMCSD_PART_LOCK_DESTROY(_part)	mtx_destroy(&(_part)->part_mtx);
-#define	MMCSD_PART_ASSERT_LOCKED(_part)					\
-	mtx_assert(&(_part)->part_mtx, MA_OWNED);
-#define	MMCSD_PART_ASSERT_UNLOCKED(_part)				\
-	mtx_assert(&(_part)->part_mtx, MA_NOTOWNED);
+#define	MMCSD_DISK_LOCK(_part)		mtx_lock(&(_part)->disk_mtx)
+#define	MMCSD_DISK_UNLOCK(_part)	mtx_unlock(&(_part)->disk_mtx)
+#define	MMCSD_DISK_LOCK_INIT(_part)					\
+	mtx_init(&(_part)->disk_mtx, (_part)->name, "mmcsd disk", MTX_DEF)
+#define	MMCSD_DISK_LOCK_DESTROY(_part)	mtx_destroy(&(_part)->disk_mtx);
+#define	MMCSD_DISK_ASSERT_LOCKED(_part)					\
+	mtx_assert(&(_part)->disk_mtx, MA_OWNED);
+#define	MMCSD_DISK_ASSERT_UNLOCKED(_part)				\
+	mtx_assert(&(_part)->disk_mtx, MA_NOTOWNED);
+
+#define	MMCSD_IOCTL_LOCK(_part)		mtx_lock(&(_part)->ioctl_mtx)
+#define	MMCSD_IOCTL_UNLOCK(_part)	mtx_unlock(&(_part)->ioctl_mtx)
+#define	MMCSD_IOCTL_LOCK_INIT(_part)					\
+	mtx_init(&(_part)->ioctl_mtx, (_part)->name, "mmcsd IOCTL", MTX_DEF)
+#define	MMCSD_IOCTL_LOCK_DESTROY(_part)	mtx_destroy(&(_part)->ioctl_mtx);
+#define	MMCSD_IOCTL_ASSERT_LOCKED(_part)				\
+	mtx_assert(&(_part)->ioctl_mtx, MA_OWNED);
+#define	MMCSD_IOCLT_ASSERT_UNLOCKED(_part)				\
+	mtx_assert(&(_part)->ioctl_mtx, MA_NOTOWNED);
 
 static int
 mmcsd_probe(device_t dev)
@@ -277,7 +292,7 @@ rtems_bsd_mmcsd_disk_read_write(struct mmcsd_part *part, rtems_blkdev_request *b
 		data_flags = MMC_DATA_READ;
 	}
 
-	MMCSD_PART_LOCK(part);
+	MMCSD_DISK_LOCK(part);
 
 	for (i = 0; i < buffer_count; ++i) {
 		rtems_blkdev_sg_buffer *sg = &blkreq->bufs [i];
@@ -354,7 +369,7 @@ rtems_bsd_mmcsd_disk_read_write(struct mmcsd_part *part, rtems_blkdev_request *b
 
 error:
 
-	MMCSD_PART_UNLOCK(part);
+	MMCSD_DISK_UNLOCK(part);
 
 	rtems_blkdev_request_done(blkreq, status_code);
 
@@ -439,6 +454,18 @@ mmcsd_attach(device_t dev)
 	sc->dev = dev;
 	sc->mmcbr = mmcbr = device_get_parent(dev);
 	sc->mode = mmcbr_get_mode(mmcbr);
+	/*
+	 * Note that in principle with an SDHCI-like re-tuning implementation,
+	 * the maximum data size can change at runtime due to a device removal/
+	 * insertion that results in switches to/from a transfer mode involving
+	 * re-tuning, iff there are multiple devices on a given bus.  Until now
+	 * mmc(4) lacks support for rescanning already attached buses, however,
+	 * and sdhci(4) to date has no support for shared buses in the first
+	 * place either.
+	 */
+	sc->max_data = mmc_get_max_data(dev);
+	sc->erase_sector = mmc_get_erase_sector(dev);
+	sc->high_cap = mmc_get_high_cap(dev);
 	sc->rca = mmc_get_rca(dev);
 
 	/* Only MMC >= 4.x devices support EXT_CSD. */
@@ -492,7 +519,7 @@ mmcsd_attach(device_t dev)
 			    (ext_csd[EXT_CSD_ENH_START_ADDR + 1] << 8) +
 			    (ext_csd[EXT_CSD_ENH_START_ADDR + 2] << 16) +
 			    (ext_csd[EXT_CSD_ENH_START_ADDR + 3] << 24)) *
-			    (mmc_get_high_cap(dev) ? MMC_SECTOR_SIZE : 1);
+			    (sc->high_cap != 0 ? MMC_SECTOR_SIZE : 1);
 		} else if (bootverbose)
 			device_printf(dev,
 			    "enhanced user data area spans entire device\n");
@@ -505,7 +532,7 @@ mmcsd_attach(device_t dev)
 	ro = mmc_get_read_only(dev);
 	mmcsd_add_part(sc, EXT_CSD_PART_CONFIG_ACC_DEFAULT, "mmcsd",
 	    device_get_unit(dev), mmc_get_media_size(dev) * sector_size,
-	    mmc_get_erase_sector(dev) * sector_size, ro);
+	    sc->erase_sector * sector_size, ro);
 
 	if (mmc_get_spec_vers(dev) < 3)
 		return (0);
@@ -644,7 +671,16 @@ mmcsd_add_part(struct mmcsd_softc *sc, u_int type, const char *name, u_int cnt,
 	part->ro = ro;
 	snprintf(part->name, sizeof(part->name), name, device_get_unit(dev));
 
-	/* For the RPMB partition, allow IOCTL access only. */
+	MMCSD_IOCTL_LOCK_INIT(part);
+
+	/*
+	 * For the RPMB partition, allow IOCTL access only.
+	 * NB: If ever attaching RPMB partitions to disk(9), the re-tuning
+	 *     implementation and especially its pausing need to be revisited,
+	 *     because then re-tuning requests may be issued by the IOCTL half
+	 *     of this driver while re-tuning is already paused by the disk(9)
+	 *     one and vice versa.
+	 */
 	if (type == EXT_CSD_PART_CONFIG_ACC_RPMB) {
 		make_dev_args_init(&args);
 		args.mda_flags = MAKEDEV_CHECKNAME | MAKEDEV_WAITOK;
@@ -659,7 +695,7 @@ mmcsd_add_part(struct mmcsd_softc *sc, u_int type, const char *name, u_int cnt,
 			return;
 		}
 	} else {
-		MMCSD_PART_LOCK_INIT(part);
+		MMCSD_DISK_LOCK_INIT(part);
 
 #ifndef __rtems__
 		d = part->disk = disk_alloc();
@@ -672,7 +708,7 @@ mmcsd_add_part(struct mmcsd_softc *sc, u_int type, const char *name, u_int cnt,
 		d->d_name = part->name;
 		d->d_drv1 = part;
 		d->d_sectorsize = mmc_get_sector_size(dev);
-		d->d_maxsize = mmc_get_max_data(dev) * d->d_sectorsize;
+		d->d_maxsize = sc->max_data * d->d_sectorsize;
 		d->d_mediasize = media_size;
 		d->d_stripesize = erase_size;
 		d->d_unit = cnt;
@@ -704,7 +740,7 @@ mmcsd_add_part(struct mmcsd_softc *sc, u_int type, const char *name, u_int cnt,
 		    part->name, cnt, bytes, unit, mmc_get_card_id_string(dev),
 		    ro ? " (read-only)" : "", device_get_nameunit(mmcbr),
 		    speed / 1000000, (speed / 100000) % 10,
-		    mmcsd_bus_bit_width(dev), mmc_get_max_data(dev));
+		    mmcsd_bus_bit_width(dev), sc->max_data);
 	} else if (type == EXT_CSD_PART_CONFIG_ACC_RPMB) {
 		printf("%s: %ju%sB partion %d%s at %s\n", part->name, bytes,
 		    unit, type, ro ? " (read-only)" : "",
@@ -795,19 +831,27 @@ mmcsd_detach(device_t dev)
 
 	for (i = 0; i < MMC_PART_MAX; i++) {
 		part = sc->part[i];
-		if (part != NULL && part->disk != NULL) {
-			MMCSD_PART_LOCK(part);
-			part->suspend = 0;
-			if (part->running > 0) {
-				/* kill thread */
-				part->running = 0;
-				wakeup(part);
-				/* wait for thread to finish. */
-				while (part->running != -1)
-					msleep(part, &part->part_mtx, 0,
-					    "detach", 0);
+		if (part != NULL) {
+			if (part->disk != NULL) {
+				MMCSD_DISK_LOCK(part);
+				part->suspend = 0;
+				if (part->running > 0) {
+					/* kill thread */
+					part->running = 0;
+					wakeup(part);
+					/* wait for thread to finish. */
+					while (part->running != -1)
+						msleep(part, &part->disk_mtx, 0,
+						    "mmcsd disk detach", 0);
+				}
+				MMCSD_DISK_UNLOCK(part);
 			}
-			MMCSD_PART_UNLOCK(part);
+			MMCSD_IOCTL_LOCK(part);
+			while (part->ioctl > 0)
+				msleep(part, &part->ioctl_mtx, 0,
+				    "mmcsd IOCTL detach", 0);
+			part->ioctl = -1;
+			MMCSD_IOCTL_UNLOCK(part);
 		}
 	}
 
@@ -823,8 +867,9 @@ mmcsd_detach(device_t dev)
 				/* kill disk */
 				disk_destroy(part->disk);
 
-				MMCSD_PART_LOCK_DESTROY(part);
+				MMCSD_DISK_LOCK_DESTROY(part);
 			}
+			MMCSD_IOCTL_LOCK_DESTROY(part);
 			free(part, M_DEVBUF);
 		}
 	}
@@ -844,19 +889,27 @@ mmcsd_suspend(device_t dev)
 
 	for (i = 0; i < MMC_PART_MAX; i++) {
 		part = sc->part[i];
-		if (part != NULL && part->disk != NULL) {
-			MMCSD_PART_LOCK(part);
-			part->suspend = 1;
-			if (part->running > 0) {
-				/* kill thread */
-				part->running = 0;
-				wakeup(part);
-				/* wait for thread to finish. */
-				while (part->running != -1)
-					msleep(part, &part->part_mtx, 0,
-					    "detach", 0);
+		if (part != NULL) {
+			if (part->disk != NULL) {
+				MMCSD_DISK_LOCK(part);
+				part->suspend = 1;
+				if (part->running > 0) {
+					/* kill thread */
+					part->running = 0;
+					wakeup(part);
+					/* wait for thread to finish. */
+					while (part->running != -1)
+						msleep(part, &part->disk_mtx, 0,
+						    "mmcsd disk suspension", 0);
+				}
+				MMCSD_DISK_UNLOCK(part);
 			}
-			MMCSD_PART_UNLOCK(part);
+			MMCSD_IOCTL_LOCK(part);
+			while (part->ioctl > 0)
+				msleep(part, &part->ioctl_mtx, 0,
+				    "mmcsd IOCTL suspension", 0);
+			part->ioctl = -1;
+			MMCSD_IOCTL_UNLOCK(part);
 		}
 	}
 #else /* __rtems__ */
@@ -875,16 +928,22 @@ mmcsd_resume(device_t dev)
 
 	for (i = 0; i < MMC_PART_MAX; i++) {
 		part = sc->part[i];
-		if (part != NULL && part->disk != NULL) {
-			MMCSD_PART_LOCK(part);
-			part->suspend = 0;
-			if (part->running <= 0) {
-				part->running = 1;
-				kproc_create(&mmcsd_task, part, &part->p, 0, 0,
-				    "%s%d: mmc/sd card", part->name, part->cnt);
-				MMCSD_PART_UNLOCK(part);
-			} else
-				MMCSD_PART_UNLOCK(part);
+		if (part != NULL) {
+			if (part->disk != NULL) {
+				MMCSD_DISK_LOCK(part);
+				part->suspend = 0;
+				if (part->running <= 0) {
+					part->running = 1;
+					MMCSD_DISK_UNLOCK(part);
+					kproc_create(&mmcsd_task, part,
+					    &part->p, 0, 0, "%s%d: mmc/sd card",
+					    part->name, part->cnt);
+				} else
+					MMCSD_DISK_UNLOCK(part);
+			}
+			MMCSD_IOCTL_LOCK(part);
+			part->ioctl = 0;
+			MMCSD_IOCTL_UNLOCK(part);
 		}
 	}
 #else /* __rtems__ */
@@ -916,13 +975,13 @@ mmcsd_strategy(struct bio *bp)
 
 	part = bp->bio_disk->d_drv1;
 	sc = part->sc;
-	MMCSD_PART_LOCK(part);
+	MMCSD_DISK_LOCK(part);
 	if (part->running > 0 || part->suspend > 0) {
 		bioq_disksort(&part->bio_queue, bp);
-		MMCSD_PART_UNLOCK(part);
+		MMCSD_DISK_UNLOCK(part);
 		wakeup(part);
 	} else {
-		MMCSD_PART_UNLOCK(part);
+		MMCSD_DISK_UNLOCK(part);
 		biofinish(bp, NULL, ENXIO);
 	}
 }
@@ -961,9 +1020,9 @@ mmcsd_ioctl(struct mmcsd_part *part, u_long cmd, void *data, int fflag)
 	switch (cmd) {
 	case MMC_IOC_CMD:
 		mic = data;
-		err = mmcsd_ioctl_cmd(part, data, fflag);
+		err = mmcsd_ioctl_cmd(part, mic, fflag);
 		break;
-	case MMC_IOC_CMD_MULTI:
+	case MMC_IOC_MULTI_CMD:
 		mimc = data;
 		if (mimc->num_of_cmds == 0)
 			break;
@@ -973,12 +1032,12 @@ mmcsd_ioctl(struct mmcsd_part *part, u_long cmd, void *data, int fflag)
 		size = sizeof(*mic) * cnt;
 		mic = malloc(size, M_TEMP, M_WAITOK);
 		err = copyin((const void *)mimc->cmds, mic, size);
-		if (err != 0)
-			break;
-		for (i = 0; i < cnt; i++) {
-			err = mmcsd_ioctl_cmd(part, &mic[i], fflag);
-			if (err != 0)
-				break;
+		if (err == 0) {
+			for (i = 0; i < cnt; i++) {
+				err = mmcsd_ioctl_cmd(part, &mic[i], fflag);
+				if (err != 0)
+					break;
+			}
 		}
 		free(mic, M_TEMP);
 		break;
@@ -1007,11 +1066,31 @@ mmcsd_ioctl_cmd(struct mmcsd_part *part, struct mmc_ioc_cmd *mic, int fflag)
 	if (part->ro == TRUE && mic->write_flag != 0)
 		return (EROFS);
 
+	/*
+	 * We don't need to explicitly lock against the disk(9) half of this
+	 * driver as MMCBUS_ACQUIRE_BUS() will serialize us.  However, it's
+	 * necessary to protect against races with detachment and suspension,
+	 * especially since it's required to switch away from RPMB partitions
+	 * again after an access (see mmcsd_switch_part()).
+	 */
+	MMCSD_IOCTL_LOCK(part);
+	while (part->ioctl != 0) {
+		if (part->ioctl < 0) {
+			MMCSD_IOCTL_UNLOCK(part);
+			return (ENXIO);
+		}
+		msleep(part, &part->ioctl_mtx, 0, "mmcsd IOCTL", 0);
+	}
+	part->ioctl = 1;
+	MMCSD_IOCTL_UNLOCK(part);
+
 	err = 0;
 	dp = NULL;
 	len = mic->blksz * mic->blocks;
-	if (len > MMC_IOC_MAX_BYTES)
-		return (EOVERFLOW);
+	if (len > MMC_IOC_MAX_BYTES) {
+		err = EOVERFLOW;
+		goto out;
+	}
 	if (len != 0) {
 		dp = malloc(len, M_TEMP, M_WAITOK);
 		err = copyin((void *)(uintptr_t)mic->data_ptr, dp, len);
@@ -1066,7 +1145,7 @@ mmcsd_ioctl_cmd(struct mmcsd_part *part, struct mmc_ioc_cmd *mic, int fflag)
 		err = mmcsd_set_blockcount(sc, mic->blocks,
 		    mic->write_flag & (1 << 31));
 		if (err != MMC_ERR_NONE)
-			goto release;
+			goto switch_back;
 	}
 	if (mic->is_acmd != 0)
 		(void)mmc_wait_for_app_cmd(mmcbr, dev, rca, &cmd, 0);
@@ -1088,6 +1167,7 @@ mmcsd_ioctl_cmd(struct mmcsd_part *part, struct mmc_ioc_cmd *mic, int fflag)
 			DELAY(1000);
 		} while (retries-- > 0);
 
+switch_back:
 		/* ... and always switch back to the default partition. */
 		err = mmcsd_switch_part(mmcbr, dev, rca,
 		    EXT_CSD_PART_CONFIG_ACC_DEFAULT);
@@ -1138,6 +1218,10 @@ release:
 	err = EIO;
 
 out:
+	MMCSD_IOCTL_LOCK(part);
+	part->ioctl = 0;
+	MMCSD_IOCTL_UNLOCK(part);
+	wakeup(part);
 	if (dp != NULL)
 		free(dp, M_TEMP);
 	return (err);
@@ -1191,10 +1275,23 @@ mmcsd_switch_part(device_t bus, device_t dev, uint16_t rca, u_int part)
 
 	sc = device_get_softc(dev);
 
-	if (sc->part_curr == part)
+	if (sc->mode == mode_sd)
 		return (MMC_ERR_NONE);
 
-	if (sc->mode == mode_sd)
+	/*
+	 * According to section "6.2.2 Command restrictions" of the eMMC
+	 * specification v5.1, CMD19/CMD21 aren't allowed to be used with
+	 * RPMB partitions.  So we pause re-tuning along with triggering
+	 * it up-front to decrease the likelihood of re-tuning becoming
+	 * necessary while accessing an RPMB partition.  Consequently, an
+	 * RPMB partition should immediately be switched away from again
+	 * after an access in order to allow for re-tuning to take place
+	 * anew.
+	 */
+	if (part == EXT_CSD_PART_CONFIG_ACC_RPMB)
+		MMCBUS_RETUNE_PAUSE(sc->mmcbr, sc->dev, true);
+
+	if (sc->part_curr == part)
 		return (MMC_ERR_NONE);
 
 	value = (sc->ext_csd[EXT_CSD_PART_CONFIG] &
@@ -1202,10 +1299,15 @@ mmcsd_switch_part(device_t bus, device_t dev, uint16_t rca, u_int part)
 	/* Jump! */
 	err = mmc_switch(bus, dev, rca, EXT_CSD_CMD_SET_NORMAL,
 	    EXT_CSD_PART_CONFIG, value, sc->part_time, true);
-	if (err != MMC_ERR_NONE)
+	if (err != MMC_ERR_NONE) {
+		if (part == EXT_CSD_PART_CONFIG_ACC_RPMB)
+			MMCBUS_RETUNE_UNPAUSE(sc->mmcbr, sc->dev);
 		return (err);
+	}
 
 	sc->ext_csd[EXT_CSD_PART_CONFIG] = value;
+	if (sc->part_curr == EXT_CSD_PART_CONFIG_ACC_RPMB)
+		MMCBUS_RETUNE_UNPAUSE(sc->mmcbr, sc->dev);
 	sc->part_curr = part;
 	return (MMC_ERR_NONE);
 }
@@ -1217,7 +1319,7 @@ mmcsd_errmsg(int e)
 
 	if (e < 0 || e > MMC_ERR_MAX)
 		return "Bad error code";
-	return errmsg[e];
+	return (errmsg[e]);
 }
 
 static daddr_t
@@ -1230,7 +1332,7 @@ mmcsd_rw(struct mmcsd_part *part, struct bio *bp)
 	struct mmc_data data;
 	struct mmcsd_softc *sc;
 	device_t dev, mmcbr;
-	int numblocks, sz;
+	u_int numblocks, sz;
 	char *vaddr;
 
 	sc = part->sc;
@@ -1242,7 +1344,7 @@ mmcsd_rw(struct mmcsd_part *part, struct bio *bp)
 	end = bp->bio_pblkno + (bp->bio_bcount / sz);
 	while (block < end) {
 		vaddr = bp->bio_data + (block - bp->bio_pblkno) * sz;
-		numblocks = min(end - block, mmc_get_max_data(dev));
+		numblocks = min(end - block, sc->max_data);
 		memset(&req, 0, sizeof(req));
 		memset(&cmd, 0, sizeof(cmd));
 		memset(&stop, 0, sizeof(stop));
@@ -1262,7 +1364,7 @@ mmcsd_rw(struct mmcsd_part *part, struct bio *bp)
 				cmd.opcode = MMC_WRITE_BLOCK;
 		}
 		cmd.arg = block;
-		if (!mmc_get_high_cap(dev))
+		if (sc->high_cap == 0)
 			cmd.arg <<= 9;
 		cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC;
 		data.data = vaddr;
@@ -1302,7 +1404,7 @@ mmcsd_delete(struct mmcsd_part *part, struct bio *bp)
 	struct mmc_request req;
 	struct mmcsd_softc *sc;
 	device_t dev, mmcbr;
-	int erase_sector, sz;
+	u_int erase_sector, sz;
 
 	sc = part->sc;
 	dev = sc->dev;
@@ -1317,7 +1419,7 @@ mmcsd_delete(struct mmcsd_part *part, struct bio *bp)
 	if (end >= part->eblock && end < part->eend)
 		end = part->eend;
 	/* Safe round to the erase sector boundaries. */
-	erase_sector = mmc_get_erase_sector(dev);
+	erase_sector = sc->erase_sector;
 	start = block + erase_sector - 1;	 /* Round up. */
 	start -= start % erase_sector;
 	stop = end;				/* Round down. */
@@ -1329,6 +1431,12 @@ mmcsd_delete(struct mmcsd_part *part, struct bio *bp)
 		return (end);
 	}
 
+	/*
+	 * Pause re-tuning so it won't interfere with the order of erase
+	 * commands.  Note that these latter don't use the data lines, so
+	 * re-tuning shouldn't actually become necessary during erase.
+	 */
+	MMCBUS_RETUNE_PAUSE(mmcbr, dev, false);
 	/* Set erase start position. */
 	memset(&req, 0, sizeof(req));
 	memset(&cmd, 0, sizeof(cmd));
@@ -1339,13 +1447,15 @@ mmcsd_delete(struct mmcsd_part *part, struct bio *bp)
 	else
 		cmd.opcode = MMC_ERASE_GROUP_START;
 	cmd.arg = start;
-	if (!mmc_get_high_cap(dev))
+	if (sc->high_cap == 0)
 		cmd.arg <<= 9;
 	cmd.flags = MMC_RSP_R1 | MMC_CMD_AC;
 	MMCBUS_WAIT_FOR_REQUEST(mmcbr, dev, &req);
 	if (req.cmd->error != MMC_ERR_NONE) {
-	    printf("erase err1: %d\n", req.cmd->error);
-	    return (block);
+		device_printf(dev, "Setting erase start position failed %d\n",
+		    req.cmd->error);
+		block = bp->bio_pblkno;
+		goto unpause;
 	}
 	/* Set erase stop position. */
 	memset(&req, 0, sizeof(req));
@@ -1356,14 +1466,16 @@ mmcsd_delete(struct mmcsd_part *part, struct bio *bp)
 	else
 		cmd.opcode = MMC_ERASE_GROUP_END;
 	cmd.arg = stop;
-	if (!mmc_get_high_cap(dev))
+	if (sc->high_cap == 0)
 		cmd.arg <<= 9;
 	cmd.arg--;
 	cmd.flags = MMC_RSP_R1 | MMC_CMD_AC;
 	MMCBUS_WAIT_FOR_REQUEST(mmcbr, dev, &req);
 	if (req.cmd->error != MMC_ERR_NONE) {
-	    printf("erase err2: %d\n", req.cmd->error);
-	    return (block);
+		device_printf(dev, "Setting erase stop position failed %d\n",
+		    req.cmd->error);
+		block = bp->bio_pblkno;
+		goto unpause;
 	}
 	/* Erase range. */
 	memset(&req, 0, sizeof(req));
@@ -1374,8 +1486,11 @@ mmcsd_delete(struct mmcsd_part *part, struct bio *bp)
 	cmd.flags = MMC_RSP_R1B | MMC_CMD_AC;
 	MMCBUS_WAIT_FOR_REQUEST(mmcbr, dev, &req);
 	if (req.cmd->error != MMC_ERR_NONE) {
-	    printf("erase err3 %d\n", req.cmd->error);
-	    return (block);
+		device_printf(dev, "erase err3: %d\n", req.cmd->error);
+		device_printf(dev, "Issuing erase command failed %d\n",
+		    req.cmd->error);
+		block = bp->bio_pblkno;
+		goto unpause;
 	}
 	/* Store one of remaining parts for the next call. */
 	if (bp->bio_pblkno >= part->eblock || block == start) {
@@ -1385,7 +1500,10 @@ mmcsd_delete(struct mmcsd_part *part, struct bio *bp)
 		part->eblock = block;	/* Predict next backward. */
 		part->eend = start;
 	}
-	return (end);
+	block = end;
+unpause:
+	MMCBUS_RETUNE_UNPAUSE(mmcbr, dev);
+	return (block);
 }
 
 static int
@@ -1446,16 +1564,16 @@ mmcsd_task(void *arg)
 	mmcbr = sc->mmcbr;
 
 	while (1) {
-		MMCSD_PART_LOCK(part);
+		MMCSD_DISK_LOCK(part);
 		do {
 			if (part->running == 0)
 				goto out;
 			bp = bioq_takefirst(&part->bio_queue);
 			if (bp == NULL)
-				msleep(part, &part->part_mtx, PRIBIO,
-				    "jobqueue", 0);
+				msleep(part, &part->disk_mtx, PRIBIO,
+				    "mmcsd disk jobqueue", 0);
 		} while (bp == NULL);
-		MMCSD_PART_UNLOCK(part);
+		MMCSD_DISK_UNLOCK(part);
 		if (bp->bio_cmd != BIO_READ && part->ro) {
 			bp->bio_error = EROFS;
 			bp->bio_resid = bp->bio_bcount;
@@ -1496,7 +1614,7 @@ release:
 out:
 	/* tell parent we're done */
 	part->running = -1;
-	MMCSD_PART_UNLOCK(part);
+	MMCSD_DISK_UNLOCK(part);
 	wakeup(part);
 
 	kproc_exit(0);
diff --git a/freebsd/sys/dev/nvme/nvme.h b/freebsd/sys/dev/nvme/nvme.h
index ff64cb00..aa640b37 100644
--- a/freebsd/sys/dev/nvme/nvme.h
+++ b/freebsd/sys/dev/nvme/nvme.h
@@ -341,9 +341,11 @@ enum nvme_admin_opcode {
 	NVME_OPC_GET_FEATURES			= 0x0a,
 	/* 0x0b - reserved */
 	NVME_OPC_ASYNC_EVENT_REQUEST		= 0x0c,
-	/* 0x0d-0x0f - reserved */
+	NVME_OPC_NAMESPACE_MANAGEMENT		= 0x0d,
+	/* 0x0e-0x0f - reserved */
 	NVME_OPC_FIRMWARE_ACTIVATE		= 0x10,
 	NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD	= 0x11,
+	NVME_OPC_NAMESPACE_ATTACHMENT		= 0x15,
 
 	NVME_OPC_FORMAT_NVM			= 0x80,
 	NVME_OPC_SECURITY_SEND			= 0x81,
@@ -456,7 +458,10 @@ struct nvme_controller_data {
 	/** maximum data transfer size */
 	uint8_t			mdts;
 
-	uint8_t			reserved1[178];
+	/** Controller ID */
+	uint16_t		ctrlr_id;
+
+	uint8_t			reserved1[176];
 
 	/* bytes 256-511: admin command set attributes */
 
@@ -471,7 +476,10 @@ struct nvme_controller_data {
 		/* supports firmware activate/download commands */
 		uint16_t	firmware  : 1;
 
-		uint16_t	oacs_rsvd : 13;
+		/* supports namespace management commands */
+		uint16_t	nsmgmt	  : 1;
+
+		uint16_t	oacs_rsvd : 12;
 	} __packed oacs;
 
 	/** abort command limit */
@@ -513,8 +521,16 @@ struct nvme_controller_data {
 		uint8_t		avscc_rsvd  : 7;
 	} __packed avscc;
 
-	uint8_t			reserved2[247];
+	uint8_t			reserved2[15];
+
+	/** Name space capabilities  */
+	struct {
+		/* if nsmgmt, report tnvmcap and unvmcap */
+		uint8_t    tnvmcap[16];
+		uint8_t    unvmcap[16];
+	} __packed untncap;
 
+	uint8_t reserved3[200];
 	/* bytes 512-703: nvm command set attributes */
 
 	/** submission queue entry size */
@@ -529,7 +545,7 @@ struct nvme_controller_data {
 		uint8_t		max : 4;
 	} __packed cqes;
 
-	uint8_t			reserved3[2];
+	uint8_t			reserved4[2];
 
 	/** number of namespaces */
 	uint32_t		nn;
@@ -555,10 +571,10 @@ struct nvme_controller_data {
 	} __packed vwc;
 
 	/* TODO: flesh out remaining nvm command set attributes */
-	uint8_t			reserved4[178];
+	uint8_t			reserved5[178];
 
 	/* bytes 704-2047: i/o command set attributes */
-	uint8_t			reserved5[1344];
+	uint8_t			reserved6[1344];
 
 	/* bytes 2048-3071: power state descriptors */
 	struct nvme_power_state power_state[32];
diff --git a/freebsd/sys/dev/rtwn/if_rtwn_rx.c b/freebsd/sys/dev/rtwn/if_rtwn_rx.c
index 5dd72605..7b3f1c2e 100644
--- a/freebsd/sys/dev/rtwn/if_rtwn_rx.c
+++ b/freebsd/sys/dev/rtwn/if_rtwn_rx.c
@@ -55,7 +55,6 @@ __FBSDID("$FreeBSD$");
 #include <dev/rtwn/if_rtwn_rx.h>
 
 #include <dev/rtwn/rtl8192c/r92c_reg.h>
-#include <dev/rtwn/rtl8192c/r92c_rx_desc.h>
 
 
 void
@@ -192,7 +191,8 @@ rtwn_get_tsf(struct rtwn_softc *sc, uint64_t *buf, int id)
 }
 
 static uint64_t
-rtwn_extend_rx_tsf(struct rtwn_softc *sc, const struct r92c_rx_stat *stat)
+rtwn_extend_rx_tsf(struct rtwn_softc *sc,
+    const struct rtwn_rx_stat_common *stat)
 {
 	uint64_t tsft;
 	uint32_t rxdw3, tsfl, tsfl_curr;
@@ -200,7 +200,7 @@ rtwn_extend_rx_tsf(struct rtwn_softc *sc, const struct r92c_rx_stat *stat)
 
 	rxdw3 = le32toh(stat->rxdw3);
 	tsfl = le32toh(stat->tsf_low);
-	id = MS(rxdw3, R92C_RXDW3_BSSID_FIT);
+	id = MS(rxdw3, RTWN_RXDW3_BSSID01_FIT);
 
 	switch (id) {
 	case 1:
@@ -243,7 +243,7 @@ rtwn_rx_common(struct rtwn_softc *sc, struct mbuf *m, void *desc)
 	struct ieee80211_frame_min *wh;
 	struct ieee80211_rx_stats rxs;
 	struct rtwn_node *un;
-	struct r92c_rx_stat *stat;
+	struct rtwn_rx_stat_common *stat;
 	void *physt;
 	uint32_t rxdw0;
 	int8_t rssi;
@@ -252,10 +252,10 @@ rtwn_rx_common(struct rtwn_softc *sc, struct mbuf *m, void *desc)
 	stat = desc;
 	rxdw0 = le32toh(stat->rxdw0);
 
-	cipher = MS(rxdw0, R92C_RXDW0_CIPHER);
-	infosz = MS(rxdw0, R92C_RXDW0_INFOSZ) * 8;
-	pktlen = MS(rxdw0, R92C_RXDW0_PKTLEN);
-	shift = MS(rxdw0, R92C_RXDW0_SHIFT);
+	cipher = MS(rxdw0, RTWN_RXDW0_CIPHER);
+	infosz = MS(rxdw0, RTWN_RXDW0_INFOSZ) * 8;
+	pktlen = MS(rxdw0, RTWN_RXDW0_PKTLEN);
+	shift = MS(rxdw0, RTWN_RXDW0_SHIFT);
 
 	wh = (struct ieee80211_frame_min *)(mtodo(m, shift + infosz));
 	if ((wh->i_fc[1] & IEEE80211_FC1_PROTECTED) &&
@@ -270,7 +270,7 @@ rtwn_rx_common(struct rtwn_softc *sc, struct mbuf *m, void *desc)
 		ni = NULL;
 	un = RTWN_NODE(ni);
 
-	if (infosz != 0 && (rxdw0 & R92C_RXDW0_PHYST))
+	if (infosz != 0 && (rxdw0 & RTWN_RXDW0_PHYST))
 		physt = (void *)mtodo(m, shift);
 	else
 		physt = (un != NULL) ? &un->last_physt : &sc->last_physt;
@@ -286,7 +286,7 @@ rtwn_rx_common(struct rtwn_softc *sc, struct mbuf *m, void *desc)
 
 	/* Add some common bits. */
 	/* NB: should not happen. */
-	if (rxdw0 & R92C_RXDW0_CRCERR)
+	if (rxdw0 & RTWN_RXDW0_CRCERR)
 		rxs.c_pktflags |= IEEE80211_RX_F_FAIL_FCSCRC;
 
 	rxs.r_flags |= IEEE80211_R_TSF_START;	/* XXX undocumented */
@@ -300,7 +300,7 @@ rtwn_rx_common(struct rtwn_softc *sc, struct mbuf *m, void *desc)
 	/* XXX TODO: we really need a rate-to-string method */
 	RTWN_DPRINTF(sc, RTWN_DEBUG_RSSI, "%s: rssi %d, rate %d\n",
 	    __func__, rssi, rxs.c_rate);
-	if (un != NULL && infosz != 0 && (rxdw0 & R92C_RXDW0_PHYST)) {
+	if (un != NULL && infosz != 0 && (rxdw0 & RTWN_RXDW0_PHYST)) {
 		/* Update our average RSSI. */
 		rtwn_update_avgrssi(sc, un, rssi, is_cck);
 	}
diff --git a/freebsd/sys/dev/rtwn/if_rtwnreg.h b/freebsd/sys/dev/rtwn/if_rtwnreg.h
index 9dc830a2..00903d88 100644
--- a/freebsd/sys/dev/rtwn/if_rtwnreg.h
+++ b/freebsd/sys/dev/rtwn/if_rtwnreg.h
@@ -18,6 +18,9 @@
  * $FreeBSD$
  */
 
+#ifndef IF_RTWNREG_H
+#define IF_RTWNREG_H
+
 #define R92C_MIN_TX_PWR		0x00
 #define R92C_MAX_TX_PWR		0x3f
 
@@ -48,6 +51,55 @@ struct rtwn_tx_desc_common {
 	} txdw7;
 } __packed __attribute__((aligned(4)));
 
+/* Common part of Rx descriptor. */
+struct rtwn_rx_stat_common {
+	uint32_t	rxdw0;
+#define RTWN_RXDW0_PKTLEN_M	0x00003fff
+#define RTWN_RXDW0_PKTLEN_S	0
+#define RTWN_RXDW0_CRCERR	0x00004000
+#define RTWN_RXDW0_ICVERR	0x00008000
+#define RTWN_RXDW0_INFOSZ_M	0x000f0000
+#define RTWN_RXDW0_INFOSZ_S	16
+#define RTWN_RXDW0_CIPHER_M	0x00700000
+#define RTWN_RXDW0_CIPHER_S	20
+#define RTWN_RXDW0_QOS		0x00800000
+#define RTWN_RXDW0_SHIFT_M	0x03000000
+#define RTWN_RXDW0_SHIFT_S	24
+#define RTWN_RXDW0_PHYST	0x04000000
+#define RTWN_RXDW0_SWDEC	0x08000000
+#define RTWN_RXDW0_LS		0x10000000
+#define RTWN_RXDW0_FS		0x20000000
+#define RTWN_RXDW0_EOR		0x40000000
+#define RTWN_RXDW0_OWN		0x80000000
+
+	uint32_t	rxdw1;
+#define RTWN_RXDW1_AMSDU	0x00002000
+#define RTWN_RXDW1_MC		0x40000000
+#define RTWN_RXDW1_BC		0x80000000
+
+	uint32_t	rxdw2;
+	uint32_t	rxdw3;
+#define RTWN_RXDW3_HTC		0x00000400
+#define RTWN_RXDW3_BSSID01_FIT_M 0x00003000
+#define RTWN_RXDW3_BSSID01_FIT_S 12
+
+	uint32_t	rxdw4;
+	uint32_t	tsf_low;
+} __packed __attribute__((aligned(4)));
+
+/* Rx descriptor for PCIe devices. */
+struct rtwn_rx_stat_pci {
+	uint32_t	rxdw0;
+	uint32_t	rxdw1;
+	uint32_t	rxdw2;
+	uint32_t	rxdw3;
+	uint32_t	rxdw4;
+	uint32_t	tsf_low;
+
+	uint32_t	rxbufaddr;
+	uint32_t	rxbufaddr64;
+} __packed __attribute__((aligned(4)));
+
 /*
  * Macros to access subfields in registers.
  */
@@ -116,3 +168,5 @@ rtwn_chan2centieee(const struct ieee80211_channel *c)
 
 	return (chan);
 }
+
+#endif	/* IF_RTWNREG_H */
diff --git a/freebsd/sys/dev/rtwn/if_rtwnvar.h b/freebsd/sys/dev/rtwn/if_rtwnvar.h
index d8754024..3ebcba52 100644
--- a/freebsd/sys/dev/rtwn/if_rtwnvar.h
+++ b/freebsd/sys/dev/rtwn/if_rtwnvar.h
@@ -25,8 +25,6 @@
 
 #define RTWN_TX_DESC_SIZE	64
 
-#define RTWN_TXBUFSZ		(16 * 1024)
-
 #define RTWN_BCN_MAX_SIZE	512
 #define RTWN_CAM_ENTRY_LIMIT	64
 
diff --git a/freebsd/sys/dev/rtwn/pci/rtwn_pci_attach.c b/freebsd/sys/dev/rtwn/pci/rtwn_pci_attach.c
index 85ede40a..c121c5a5 100644
--- a/freebsd/sys/dev/rtwn/pci/rtwn_pci_attach.c
+++ b/freebsd/sys/dev/rtwn/pci/rtwn_pci_attach.c
@@ -64,7 +64,6 @@ __FBSDID("$FreeBSD$");
 #include <dev/rtwn/pci/rtwn_pci_tx.h>
 
 #include <dev/rtwn/rtl8192c/pci/r92ce_reg.h>
-#include <dev/rtwn/rtl8192c/pci/r92ce_rx_desc.h>
 
 
 static device_probe_t	rtwn_pci_probe;
@@ -135,7 +134,7 @@ rtwn_pci_alloc_rx_list(struct rtwn_softc *sc)
 	int i, error;
 
 	/* Allocate Rx descriptors. */
-	size = sizeof(struct r92ce_rx_stat) * RTWN_PCI_RX_LIST_COUNT;
+	size = sizeof(struct rtwn_rx_stat_pci) * RTWN_PCI_RX_LIST_COUNT;
 	error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), 1, 0,
 	    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL,
 	    size, 1, size, 0, NULL, NULL, &rx_ring->desc_dmat);
diff --git a/freebsd/sys/dev/rtwn/pci/rtwn_pci_rx.c b/freebsd/sys/dev/rtwn/pci/rtwn_pci_rx.c
index 150500d8..1934b741 100644
--- a/freebsd/sys/dev/rtwn/pci/rtwn_pci_rx.c
+++ b/freebsd/sys/dev/rtwn/pci/rtwn_pci_rx.c
@@ -58,8 +58,6 @@ __FBSDID("$FreeBSD$");
 #include <dev/rtwn/pci/rtwn_pci_var.h>
 #include <dev/rtwn/pci/rtwn_pci_rx.h>
 
-#include <dev/rtwn/rtl8192c/pci/r92ce_rx_desc.h>
-
 
 void
 rtwn_pci_dma_map_addr(void *arg, bus_dma_segment_t *segs, int nsegs,
@@ -73,21 +71,21 @@ rtwn_pci_dma_map_addr(void *arg, bus_dma_segment_t *segs, int nsegs,
 }
 
 void
-rtwn_pci_setup_rx_desc(struct rtwn_pci_softc *pc, struct r92ce_rx_stat *desc,
-    bus_addr_t addr, size_t len, int idx)
+rtwn_pci_setup_rx_desc(struct rtwn_pci_softc *pc,
+    struct rtwn_rx_stat_pci *desc, bus_addr_t addr, size_t len, int idx)
 {
 
 	memset(desc, 0, sizeof(*desc));
-	desc->rxdw0 = htole32(SM(R92C_RXDW0_PKTLEN, len) |
-		((idx == RTWN_PCI_RX_LIST_COUNT - 1) ? R92C_RXDW0_EOR : 0));
+	desc->rxdw0 = htole32(SM(RTWN_RXDW0_PKTLEN, len) |
+		((idx == RTWN_PCI_RX_LIST_COUNT - 1) ? RTWN_RXDW0_EOR : 0));
 	desc->rxbufaddr = htole32(addr);
 	bus_space_barrier(pc->pc_st, pc->pc_sh, 0, pc->pc_mapsize,
 	    BUS_SPACE_BARRIER_WRITE);
-	desc->rxdw0 |= htole32(R92C_RXDW0_OWN);
+	desc->rxdw0 |= htole32(RTWN_RXDW0_OWN);
 }
 
 static void
-rtwn_pci_rx_frame(struct rtwn_softc *sc, struct r92ce_rx_stat *rx_desc,
+rtwn_pci_rx_frame(struct rtwn_softc *sc, struct rtwn_rx_stat_pci *rx_desc,
     int desc_idx)
 {
 	struct rtwn_pci_softc *pc = RTWN_PCI_SOFTC(sc);
@@ -109,18 +107,18 @@ rtwn_pci_rx_frame(struct rtwn_softc *sc, struct r92ce_rx_stat *rx_desc,
 	    le32toh(rx_desc->rxbufaddr), le32toh(rx_desc->rxbufaddr64));
 
 	rxdw0 = le32toh(rx_desc->rxdw0);
-	if (__predict_false(rxdw0 & (R92C_RXDW0_CRCERR | R92C_RXDW0_ICVERR))) {
+	if (__predict_false(rxdw0 & (RTWN_RXDW0_CRCERR | RTWN_RXDW0_ICVERR))) {
 		/*
 		 * This should not happen since we setup our Rx filter
 		 * to not receive these frames.
 		 */
 		RTWN_DPRINTF(sc, RTWN_DEBUG_RECV,
 		    "%s: RX flags error (%s)\n", __func__,
-		    rxdw0 & R92C_RXDW0_CRCERR ? "CRC" : "ICV");
+		    rxdw0 & RTWN_RXDW0_CRCERR ? "CRC" : "ICV");
 		goto fail;
 	}
 
-	pktlen = MS(rxdw0, R92C_RXDW0_PKTLEN);
+	pktlen = MS(rxdw0, RTWN_RXDW0_PKTLEN);
 	if (__predict_false(pktlen < sizeof(struct ieee80211_frame_ack) ||
 	    pktlen > MJUMPAGESIZE)) {
 		RTWN_DPRINTF(sc, RTWN_DEBUG_RECV,
@@ -128,8 +126,8 @@ rtwn_pci_rx_frame(struct rtwn_softc *sc, struct r92ce_rx_stat *rx_desc,
 		goto fail;
 	}
 
-	infosz = MS(rxdw0, R92C_RXDW0_INFOSZ) * 8;
-	shift = MS(rxdw0, R92C_RXDW0_SHIFT);
+	infosz = MS(rxdw0, RTWN_RXDW0_INFOSZ) * 8;
+	shift = MS(rxdw0, RTWN_RXDW0_SHIFT);
 
 	m1 = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, MJUMPAGESIZE);
 	if (__predict_false(m1 == NULL)) {
@@ -270,9 +268,9 @@ rtwn_pci_rx_done(struct rtwn_softc *sc)
 	bus_dmamap_sync(ring->desc_dmat, ring->desc_map, BUS_DMASYNC_POSTREAD);
 
 	for (;;) {
-		struct r92ce_rx_stat *rx_desc = &ring->desc[ring->cur];
+		struct rtwn_rx_stat_pci *rx_desc = &ring->desc[ring->cur];
 
-		if (le32toh(rx_desc->rxdw0) & R92C_RXDW0_OWN)
+		if (le32toh(rx_desc->rxdw0) & RTWN_RXDW0_OWN)
 			break;
 
 		rtwn_pci_rx_frame(sc, rx_desc, ring->cur);
diff --git a/freebsd/sys/dev/rtwn/pci/rtwn_pci_rx.h b/freebsd/sys/dev/rtwn/pci/rtwn_pci_rx.h
index 265d32d8..30dd785a 100644
--- a/freebsd/sys/dev/rtwn/pci/rtwn_pci_rx.h
+++ b/freebsd/sys/dev/rtwn/pci/rtwn_pci_rx.h
@@ -21,7 +21,7 @@
 
 void	rtwn_pci_dma_map_addr(void *, bus_dma_segment_t *, int, int);
 void	rtwn_pci_setup_rx_desc(struct rtwn_pci_softc *,
-	    struct r92ce_rx_stat *, bus_addr_t, size_t, int);
+	    struct rtwn_rx_stat_pci *, bus_addr_t, size_t, int);
 void	rtwn_pci_intr(void *);
 
 #endif	/* RTWN_PCI_RX_H */
diff --git a/freebsd/sys/dev/rtwn/pci/rtwn_pci_var.h b/freebsd/sys/dev/rtwn/pci/rtwn_pci_var.h
index 5a9e64e7..194fab4a 100644
--- a/freebsd/sys/dev/rtwn/pci/rtwn_pci_var.h
+++ b/freebsd/sys/dev/rtwn/pci/rtwn_pci_var.h
@@ -23,9 +23,6 @@
 #ifndef RTWN_PCI_VAR_H
 #define RTWN_PCI_VAR_H
 
-#include <dev/rtwn/rtl8192c/pci/r92ce_rx_desc.h>
-
-
 #define RTWN_PCI_RX_LIST_COUNT		256
 #define RTWN_PCI_TX_LIST_COUNT		256
 
@@ -36,7 +33,7 @@ struct rtwn_rx_data {
 };
 
 struct rtwn_rx_ring {
-	struct r92ce_rx_stat	*desc;
+	struct rtwn_rx_stat_pci	*desc;
 	bus_addr_t		paddr;
 	bus_dma_tag_t		desc_dmat;
 	bus_dmamap_t		desc_map;
diff --git a/freebsd/sys/dev/rtwn/rtl8188e/r88e_chan.c b/freebsd/sys/dev/rtwn/rtl8188e/r88e_chan.c
index fe9d58b7..903398a4 100644
--- a/freebsd/sys/dev/rtwn/rtl8188e/r88e_chan.c
+++ b/freebsd/sys/dev/rtwn/rtl8188e/r88e_chan.c
@@ -91,8 +91,7 @@ r88e_get_txpower(struct rtwn_softc *sc, int chain,
 {
 	struct r92c_softc *rs = sc->sc_priv;
 	const struct rtwn_r88e_txpwr *rt = rs->rs_txpwr;
-	const struct rtwn_r88e_txagc *base = rs->rs_txagc;
-	uint16_t cckpow, ofdmpow, bw20pow, htpow;
+	uint8_t cckpow, ofdmpow, bw20pow, htpow = 0;
 	int max_mcs, ridx, group;
 
 	/* Determine channel group. */
@@ -108,35 +107,24 @@ r88e_get_txpower(struct rtwn_softc *sc, int chain,
 	KASSERT(max_mcs <= RTWN_RIDX_COUNT, ("increase ridx limit\n"));
 
 	memset(power, 0, max_mcs * sizeof(power[0]));
-	if (rs->regulatory == 0) {
-		for (ridx = RTWN_RIDX_CCK1; ridx <= RTWN_RIDX_CCK11; ridx++)
-			power[ridx] = base->pwr[0][ridx];
-	}
-	for (ridx = RTWN_RIDX_OFDM6; ridx <= max_mcs; ridx++) {
-		if (rs->regulatory == 3)
-			power[ridx] = base->pwr[0][ridx];
-		else if (rs->regulatory == 1) {
-			if (!IEEE80211_IS_CHAN_HT40(c))
-				power[ridx] = base->pwr[group][ridx];
-		} else if (rs->regulatory != 2)
-			power[ridx] = base->pwr[0][ridx];
-	}
 
 	/* Compute per-CCK rate Tx power. */
 	cckpow = rt->cck_tx_pwr[group];
-	for (ridx = RTWN_RIDX_CCK1; ridx <= RTWN_RIDX_CCK11; ridx++)
-		power[ridx] += cckpow;
+	for (ridx = RTWN_RIDX_CCK1; ridx <= RTWN_RIDX_CCK11; ridx++) {
+		power[ridx] = (ridx == RTWN_RIDX_CCK2) ? cckpow - 9 : cckpow;
+	}
 
-	htpow = rt->ht40_tx_pwr[group];
+	if (group < 5)
+		htpow = rt->ht40_tx_pwr[group];
 
 	/* Compute per-OFDM rate Tx power. */
 	ofdmpow = htpow + rt->ofdm_tx_pwr_diff;
 	for (ridx = RTWN_RIDX_OFDM6; ridx <= RTWN_RIDX_OFDM54; ridx++)
-		power[ridx] += ofdmpow;
+		power[ridx] = ofdmpow;
 
 	bw20pow = htpow + rt->bw20_tx_pwr_diff;
 	for (ridx = RTWN_RIDX_MCS(0); ridx <= max_mcs; ridx++)
-		power[ridx] += bw20pow;
+		power[ridx] = bw20pow;
 
 	/* Apply max limit. */
 	for (ridx = RTWN_RIDX_CCK1; ridx <= max_mcs; ridx++) {
diff --git a/freebsd/sys/dev/rtwn/rtl8188e/r88e_priv.h b/freebsd/sys/dev/rtwn/rtl8188e/r88e_priv.h
index cb4f7edb..28f4b1fb 100644
--- a/freebsd/sys/dev/rtwn/rtl8188e/r88e_priv.h
+++ b/freebsd/sys/dev/rtwn/rtl8188e/r88e_priv.h
@@ -227,47 +227,4 @@ static const struct rtwn_rf_prog rtl8188eu_rf[] = {
 	{ 0, NULL, NULL, { 0 }, NULL }
 };
 
-
-struct rtwn_r88e_txagc {
-	uint8_t	pwr[R88E_GROUP_2G][20];	/* RTWN_RIDX_MCS(7) + 1 */
-};
-
-/*
- * Per RF chain/group/rate Tx gain values.
- */
-static const struct rtwn_r88e_txagc r88e_txagc[] = {
-	{ {	/* Chain 0. */
-	{	/* Group 0. */
-	0x00, 0x00, 0x00, 0x00,				/* CCK1~11. */
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,	/* OFDM6~54. */
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,	/* MCS0~7. */
-	},
-	{	/* Group 1. */
-	0x00, 0x00, 0x00, 0x00,				/* CCK1~11. */
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,	/* OFDM6~54. */
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,	/* MCS0~7. */
-	},
-	{	/* Group 2. */
-	0x00, 0x00, 0x00, 0x00,				/* CCK1~11. */
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,	/* OFDM6~54. */
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,	/* MCS0~7. */
-	},
-	{	/* Group 3. */
-	0x00, 0x00, 0x00, 0x00,				/* CCK1~11. */
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,	/* OFDM6~54. */
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,	/* MCS0~7. */
-	},
-	{	/* Group 4. */
-	0x00, 0x00, 0x00, 0x00,				/* CCK1~11. */
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,	/* OFDM6~54. */
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,	/* MCS0~7. */
-	},
-	{	/* Group 5. */
-	0x00, 0x00, 0x00, 0x00,				/* CCK1~11. */
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,	/* OFDM6~54. */
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,	/* MCS0~7. */
-	}
-	} }
-};
-
 #endif	/* R88E_PRIV_H */
diff --git a/freebsd/sys/dev/rtwn/rtl8188e/r88e_rom_defs.h b/freebsd/sys/dev/rtwn/rtl8188e/r88e_rom_defs.h
index 5734246f..c6033678 100644
--- a/freebsd/sys/dev/rtwn/rtl8188e/r88e_rom_defs.h
+++ b/freebsd/sys/dev/rtwn/rtl8188e/r88e_rom_defs.h
@@ -23,7 +23,7 @@
 
 #define R88E_GROUP_2G		6
 
-#define R88E_EFUSE_MAX_LEN	512
+#define R88E_EFUSE_MAX_LEN	256
 #define R88E_EFUSE_MAP_LEN	512
 
 #endif	/* R88E_ROM_DEFS_H */
diff --git a/freebsd/sys/dev/rtwn/rtl8188e/r88e_rx.c b/freebsd/sys/dev/rtwn/rtl8188e/r88e_rx.c
index 88159876..856ec88b 100644
--- a/freebsd/sys/dev/rtwn/rtl8188e/r88e_rx.c
+++ b/freebsd/sys/dev/rtwn/rtl8188e/r88e_rx.c
@@ -223,7 +223,7 @@ r88e_get_rx_stats(struct rtwn_softc *sc, struct ieee80211_rx_stats *rxs,
 
 	if (!sc->sc_ht40) {	/* XXX center channel */
 		rxs->r_flags |= IEEE80211_R_IEEE | IEEE80211_R_FREQ;
-		rxs->c_ieee = le16toh(physt->chan);
+		rxs->c_ieee = physt->chan;
 		rxs->c_freq = ieee80211_ieee2mhz(rxs->c_ieee,
 		    IEEE80211_CHAN_2GHZ);
 	}
diff --git a/freebsd/sys/dev/rtwn/rtl8188e/usb/r88eu_attach.c b/freebsd/sys/dev/rtwn/rtl8188e/usb/r88eu_attach.c
index f5ac1d9d..73cc7856 100644
--- a/freebsd/sys/dev/rtwn/rtl8188e/usb/r88eu_attach.c
+++ b/freebsd/sys/dev/rtwn/rtl8188e/usb/r88eu_attach.c
@@ -87,7 +87,7 @@ r88eu_attach_private(struct rtwn_softc *sc)
 	rs = malloc(sizeof(struct r92c_softc), M_RTWN_PRIV, M_WAITOK | M_ZERO);
 
 	rs->rs_txpwr			= &r88e_txpwr;
-	rs->rs_txagc			= &r88e_txagc;
+	rs->rs_txagc			= NULL;
 
 	rs->rs_set_bw20			= r88e_set_bw20;
 	rs->rs_get_txpower		= r88e_get_txpower;
diff --git a/freebsd/sys/dev/rtwn/rtl8192c/r92c_rom_image.h b/freebsd/sys/dev/rtwn/rtl8192c/r92c_rom_image.h
index 304324e6..5c2880bb 100644
--- a/freebsd/sys/dev/rtwn/rtl8192c/r92c_rom_image.h
+++ b/freebsd/sys/dev/rtwn/rtl8192c/r92c_rom_image.h
@@ -48,7 +48,7 @@ struct r92c_rom {
 	uint8_t		ofdm_tx_pwr_diff[R92C_GROUP_2G];
 	uint8_t		ht40_max_pwr[R92C_GROUP_2G];
 	uint8_t		ht20_max_pwr[R92C_GROUP_2G];
-	uint8_t		xtal_calib;
+	uint8_t		channel_plan;
 	uint8_t		tssi[R92C_MAX_CHAINS];
 	uint8_t		thermal_meter;
 #define R92C_ROM_THERMAL_METER_M	0x1f
@@ -58,9 +58,7 @@ struct r92c_rom {
 	uint8_t		rf_opt2;
 	uint8_t		rf_opt3;
 	uint8_t		rf_opt4;
-	uint8_t		channel_plan;
-#define R92C_CHANNEL_PLAN_BY_HW		0x80
-
+	uint8_t		reserved5;
 	uint8_t		version;
 	uint8_t		customer_id;
 } __packed;
diff --git a/freebsd/sys/dev/rtwn/rtl8192c/r92c_tx.c b/freebsd/sys/dev/rtwn/rtl8192c/r92c_tx.c
index 454da87c..c2a6eab0 100644
--- a/freebsd/sys/dev/rtwn/rtl8192c/r92c_tx.c
+++ b/freebsd/sys/dev/rtwn/rtl8192c/r92c_tx.c
@@ -333,8 +333,6 @@ r92c_fill_tx_desc(struct rtwn_softc *sc, struct ieee80211_node *ni,
 
 		if (m->m_flags & M_AMPDU_MPDU) {
 			seqno = ni->ni_txseqs[tid];
-			/* NB: clear Fragment Number field. */
-			*(uint16_t *)wh->i_seq = 0;
 			ni->ni_txseqs[tid]++;
 		} else
 			seqno = M_SEQNO_GET(m) % IEEE80211_SEQ_RANGE;
diff --git a/freebsd/sys/dev/rtwn/rtl8812a/r12a_tx.c b/freebsd/sys/dev/rtwn/rtl8812a/r12a_tx.c
index c5a9e465..895f71e4 100644
--- a/freebsd/sys/dev/rtwn/rtl8812a/r12a_tx.c
+++ b/freebsd/sys/dev/rtwn/rtl8812a/r12a_tx.c
@@ -340,8 +340,6 @@ r12a_fill_tx_desc(struct rtwn_softc *sc, struct ieee80211_node *ni,
 
 		if (m->m_flags & M_AMPDU_MPDU) {
 			seqno = ni->ni_txseqs[tid];
-			/* NB: clear Fragment Number field. */
-			*(uint16_t *)wh->i_seq = 0;
 			ni->ni_txseqs[tid]++;
 		} else
 			seqno = M_SEQNO_GET(m) % IEEE80211_SEQ_RANGE;
diff --git a/freebsd/sys/dev/rtwn/usb/rtwn_usb_attach.c b/freebsd/sys/dev/rtwn/usb/rtwn_usb_attach.c
index 93e1f768..8626d0a3 100644
--- a/freebsd/sys/dev/rtwn/usb/rtwn_usb_attach.c
+++ b/freebsd/sys/dev/rtwn/usb/rtwn_usb_attach.c
@@ -79,12 +79,14 @@ static void	rtwn_usb_reset_lists(struct rtwn_softc *,
 		    struct ieee80211vap *);
 static void	rtwn_usb_reset_tx_list(struct rtwn_usb_softc *,
 		    rtwn_datahead *, struct ieee80211vap *);
+static void	rtwn_usb_reset_rx_list(struct rtwn_usb_softc *);
 static void	rtwn_usb_start_xfers(struct rtwn_softc *);
 static void	rtwn_usb_abort_xfers(struct rtwn_softc *);
 static int	rtwn_usb_fw_write_block(struct rtwn_softc *,
 		    const uint8_t *, uint16_t, int);
 static void	rtwn_usb_drop_incorrect_tx(struct rtwn_softc *);
 static void	rtwn_usb_attach_methods(struct rtwn_softc *);
+static void	rtwn_usb_sysctlattach(struct rtwn_softc *);
 
 #define RTWN_CONFIG_INDEX	0
 
@@ -135,9 +137,8 @@ rtwn_usb_alloc_rx_list(struct rtwn_softc *sc)
 	struct rtwn_usb_softc *uc = RTWN_USB_SOFTC(sc);
 	int error, i;
 
-	/* XXX recheck */
 	error = rtwn_usb_alloc_list(sc, uc->uc_rx, RTWN_USB_RX_LIST_COUNT,
-	    sc->rx_dma_size + 1024);
+	    uc->uc_rx_buf_size * RTWN_USB_RXBUFSZ_UNIT);
 	if (error != 0)
 		return (error);
 
@@ -157,7 +158,7 @@ rtwn_usb_alloc_tx_list(struct rtwn_softc *sc)
 	int error, i;
 
 	error = rtwn_usb_alloc_list(sc, uc->uc_tx, RTWN_USB_TX_LIST_COUNT,
-	    RTWN_TXBUFSZ);
+	    RTWN_USB_TXBUFSZ);
 	if (error != 0)
 		return (error);
 
@@ -201,6 +202,9 @@ rtwn_usb_free_rx_list(struct rtwn_softc *sc)
 
 	rtwn_usb_free_list(sc, uc->uc_rx, RTWN_USB_RX_LIST_COUNT);
 
+	uc->uc_rx_stat_len = 0;
+	uc->uc_rx_off = 0;
+
 	STAILQ_INIT(&uc->uc_rx_active);
 	STAILQ_INIT(&uc->uc_rx_inactive);
 }
@@ -226,8 +230,10 @@ rtwn_usb_reset_lists(struct rtwn_softc *sc, struct ieee80211vap *vap)
 
 	rtwn_usb_reset_tx_list(uc, &uc->uc_tx_active, vap);
 	rtwn_usb_reset_tx_list(uc, &uc->uc_tx_pending, vap);
-	if (vap == NULL)
+	if (vap == NULL) {
+		rtwn_usb_reset_rx_list(uc);
 		sc->qfullmsk = 0;
+	}
 }
 
 static void
@@ -261,6 +267,23 @@ rtwn_usb_reset_tx_list(struct rtwn_usb_softc *uc,
 }
 
 static void
+rtwn_usb_reset_rx_list(struct rtwn_usb_softc *uc)
+{
+	int i;
+
+	for (i = 0; i < RTWN_USB_RX_LIST_COUNT; i++) {
+		struct rtwn_data *dp = &uc->uc_rx[i];
+
+		if (dp->m != NULL) {
+			m_freem(dp->m);
+			dp->m = NULL;
+		}
+	}
+	uc->uc_rx_stat_len = 0;
+	uc->uc_rx_off = 0;
+}
+
+static void
 rtwn_usb_start_xfers(struct rtwn_softc *sc)
 {
 	struct rtwn_usb_softc *uc = RTWN_USB_SOFTC(sc);
@@ -329,6 +352,31 @@ rtwn_usb_attach_methods(struct rtwn_softc *sc)
 	sc->bcn_check_interval	= 100;
 }
 
+static void
+rtwn_usb_sysctlattach(struct rtwn_softc *sc)
+{
+	struct rtwn_usb_softc *uc = RTWN_USB_SOFTC(sc);
+	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->sc_dev);
+	struct sysctl_oid *tree = device_get_sysctl_tree(sc->sc_dev);
+	char str[64];
+	int ret;
+
+	ret = snprintf(str, sizeof(str),
+	    "Rx buffer size, 512-byte units [%d...%d]",
+	    RTWN_USB_RXBUFSZ_MIN, RTWN_USB_RXBUFSZ_MAX);
+	KASSERT(ret > 0, ("ret (%d) <= 0!\n", ret));
+	(void) ret;
+
+	uc->uc_rx_buf_size = RTWN_USB_RXBUFSZ_DEF;
+	SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
+	    "rx_buf_size", CTLFLAG_RDTUN, &uc->uc_rx_buf_size,
+	    uc->uc_rx_buf_size, str);
+	if (uc->uc_rx_buf_size < RTWN_USB_RXBUFSZ_MIN)
+		uc->uc_rx_buf_size = RTWN_USB_RXBUFSZ_MIN;
+	if (uc->uc_rx_buf_size > RTWN_USB_RXBUFSZ_MAX)
+		uc->uc_rx_buf_size = RTWN_USB_RXBUFSZ_MAX;
+}
+
 static int
 rtwn_usb_attach(device_t self)
 {
@@ -345,6 +393,7 @@ rtwn_usb_attach(device_t self)
 
 	/* Need to be initialized early. */
 	rtwn_sysctlattach(sc);
+	rtwn_usb_sysctlattach(sc);
 	mtx_init(&sc->sc_mtx, ic->ic_name, MTX_NETWORK_LOCK, MTX_DEF);
 
 	rtwn_usb_attach_methods(sc);
diff --git a/freebsd/sys/dev/rtwn/usb/rtwn_usb_ep.c b/freebsd/sys/dev/rtwn/usb/rtwn_usb_ep.c
index ef7d1ffc..f56e96c0 100644
--- a/freebsd/sys/dev/rtwn/usb/rtwn_usb_ep.c
+++ b/freebsd/sys/dev/rtwn/usb/rtwn_usb_ep.c
@@ -75,7 +75,7 @@ static const struct usb_config rtwn_config_common[RTWN_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.bufsize = RTWN_TXBUFSZ,
+		.bufsize = RTWN_USB_TXBUFSZ,
 		.flags = {
 			.ext_buffer = 1,
 			.pipe_bof = 1,
@@ -88,7 +88,7 @@ static const struct usb_config rtwn_config_common[RTWN_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.bufsize = RTWN_TXBUFSZ,
+		.bufsize = RTWN_USB_TXBUFSZ,
 		.flags = {
 			.ext_buffer = 1,
 			.pipe_bof = 1,
@@ -101,7 +101,7 @@ static const struct usb_config rtwn_config_common[RTWN_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.bufsize = RTWN_TXBUFSZ,
+		.bufsize = RTWN_USB_TXBUFSZ,
 		.flags = {
 			.ext_buffer = 1,
 			.pipe_bof = 1,
@@ -114,7 +114,7 @@ static const struct usb_config rtwn_config_common[RTWN_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.bufsize = RTWN_TXBUFSZ,
+		.bufsize = RTWN_USB_TXBUFSZ,
 		.flags = {
 			.ext_buffer = 1,
 			.pipe_bof = 1,
@@ -227,7 +227,8 @@ rtwn_usb_setup_endpoints(struct rtwn_usb_softc *uc)
 		break;
 	}
 
-	rtwn_config[RTWN_BULK_RX].bufsize = sc->rx_dma_size + 1024;
+	rtwn_config[RTWN_BULK_RX].bufsize =
+	    uc->uc_rx_buf_size * RTWN_USB_RXBUFSZ_UNIT;
 	error = usbd_transfer_setup(uc->uc_udev, &iface_index,
 	    uc->uc_xfer, rtwn_config, RTWN_N_TRANSFER, uc, &sc->sc_mtx);
 	free(rtwn_config, M_TEMP);
diff --git a/freebsd/sys/dev/rtwn/usb/rtwn_usb_rx.c b/freebsd/sys/dev/rtwn/usb/rtwn_usb_rx.c
index 4f39b580..9de8fca4 100644
--- a/freebsd/sys/dev/rtwn/usb/rtwn_usb_rx.c
+++ b/freebsd/sys/dev/rtwn/usb/rtwn_usb_rx.c
@@ -65,57 +65,69 @@ __FBSDID("$FreeBSD$");
 #include <dev/rtwn/usb/rtwn_usb_var.h>
 #include <dev/rtwn/usb/rtwn_usb_rx.h>
 
-#include <dev/rtwn/rtl8192c/r92c_reg.h>	/* for CAM_ALGO_NONE */
-#include <dev/rtwn/rtl8192c/r92c_rx_desc.h>
+static struct mbuf *	rtwn_rxeof(struct rtwn_softc *, struct rtwn_data *,
+			    uint8_t *, int);
 
-
-static struct mbuf *
-rtwn_rx_copy_to_mbuf(struct rtwn_softc *sc, struct r92c_rx_stat *stat,
-    int totlen)
+static int
+rtwn_rx_check_pre_alloc(struct rtwn_softc *sc,
+    struct rtwn_rx_stat_common *stat)
 {
-	struct ieee80211com *ic = &sc->sc_ic;
-	struct mbuf *m;
 	uint32_t rxdw0;
 	int pktlen;
 
 	RTWN_ASSERT_LOCKED(sc);
 
-	/* Dump Rx descriptor. */
-	RTWN_DPRINTF(sc, RTWN_DEBUG_RECV_DESC,
-	    "%s: dw: 0 %08X, 1 %08X, 2 %08X, 3 %08X, 4 %08X, tsfl %08X\n",
-	    __func__, le32toh(stat->rxdw0), le32toh(stat->rxdw1),
-	    le32toh(stat->rxdw2), le32toh(stat->rxdw3), le32toh(stat->rxdw4),
-	    le32toh(stat->tsf_low));
-
 	/*
 	 * don't pass packets to the ieee80211 framework if the driver isn't
 	 * RUNNING.
 	 */
 	if (!(sc->sc_flags & RTWN_RUNNING))
-		return (NULL);
+		return (-1);
 
 	rxdw0 = le32toh(stat->rxdw0);
-	if (__predict_false(rxdw0 & (R92C_RXDW0_CRCERR | R92C_RXDW0_ICVERR))) {
+	if (__predict_false(rxdw0 & (RTWN_RXDW0_CRCERR | RTWN_RXDW0_ICVERR))) {
 		/*
 		 * This should not happen since we setup our Rx filter
 		 * to not receive these frames.
 		 */
 		RTWN_DPRINTF(sc, RTWN_DEBUG_RECV,
 		    "%s: RX flags error (%s)\n", __func__,
-		    rxdw0 & R92C_RXDW0_CRCERR ? "CRC" : "ICV");
-		goto fail;
+		    rxdw0 & RTWN_RXDW0_CRCERR ? "CRC" : "ICV");
+		return (-1);
 	}
 
-	pktlen = MS(rxdw0, R92C_RXDW0_PKTLEN);
+	pktlen = MS(rxdw0, RTWN_RXDW0_PKTLEN);
 	if (__predict_false(pktlen < sizeof(struct ieee80211_frame_ack))) {
 		/*
 		 * Should not happen (because of Rx filter setup).
 		 */
 		RTWN_DPRINTF(sc, RTWN_DEBUG_RECV,
 		    "%s: frame is too short: %d\n", __func__, pktlen);
-		goto fail;
+		return (-1);
 	}
 
+	return (0);
+}
+
+static struct mbuf *
+rtwn_rx_copy_to_mbuf(struct rtwn_softc *sc, struct rtwn_rx_stat_common *stat,
+    int totlen)
+{
+	struct ieee80211com *ic = &sc->sc_ic;
+	struct mbuf *m;
+
+	RTWN_ASSERT_LOCKED(sc);
+
+	/* Dump Rx descriptor. */
+	RTWN_DPRINTF(sc, RTWN_DEBUG_RECV_DESC,
+	    "%s: dw: 0 %08X, 1 %08X, 2 %08X, 3 %08X, 4 %08X, tsfl %08X\n",
+	    __func__, le32toh(stat->rxdw0), le32toh(stat->rxdw1),
+	    le32toh(stat->rxdw2), le32toh(stat->rxdw3), le32toh(stat->rxdw4),
+	    le32toh(stat->tsf_low));
+
+	if (rtwn_rx_check_pre_alloc(sc, stat) != 0)
+		goto fail;
+
 	m = m_get2(totlen, M_NOWAIT, MT_DATA, M_PKTHDR);
 	if (__predict_false(m == NULL)) {
 		device_printf(sc->sc_dev, "%s: could not allocate RX mbuf\n",
@@ -139,30 +151,124 @@ fail:
 }
 
 static struct mbuf *
-rtwn_rxeof(struct rtwn_softc *sc, uint8_t *buf, int len)
+rtwn_rxeof_fragmented(struct rtwn_usb_softc *uc, struct rtwn_data *data,
+    uint8_t *buf, int len)
+{
+	struct rtwn_softc *sc = &uc->uc_sc;
+	struct ieee80211com *ic = &sc->sc_ic;
+	struct rtwn_rx_stat_common *stat = &uc->uc_rx_stat;
+	uint32_t rxdw0;
+	int totlen, pktlen, infosz, min_len;
+	int orig_len = len;
+	int alloc_mbuf = 0;
+
+	/* Check if Rx descriptor is not truncated. */
+	if (uc->uc_rx_stat_len < sizeof(*stat)) {
+		min_len = min(sizeof(*stat) - uc->uc_rx_stat_len, len);
+		memcpy((uint8_t *)stat + uc->uc_rx_stat_len, buf, min_len);
+
+		uc->uc_rx_stat_len += min_len;
+		buf += min_len;
+		len -= min_len;
+
+		if (uc->uc_rx_stat_len < sizeof(*stat))
+			goto end;
+
+		KASSERT(data->m == NULL, ("%s: data->m != NULL!\n", __func__));
+		alloc_mbuf = 1;
+
+		/* Dump Rx descriptor. */
+		RTWN_DPRINTF(sc, RTWN_DEBUG_RECV_DESC,
+		    "%s: dw: 0 %08X, 1 %08X, 2 %08X, 3 %08X, 4 %08X, "
+		    "tsfl %08X\n", __func__, le32toh(stat->rxdw0),
+		    le32toh(stat->rxdw1), le32toh(stat->rxdw2),
+		    le32toh(stat->rxdw3), le32toh(stat->rxdw4),
+		    le32toh(stat->tsf_low));
+	}
+
+	rxdw0 = le32toh(stat->rxdw0);
+	pktlen = MS(rxdw0, RTWN_RXDW0_PKTLEN);
+	infosz = MS(rxdw0, RTWN_RXDW0_INFOSZ) * 8;
+	totlen = sizeof(*stat) + infosz + pktlen;
+	if (alloc_mbuf) {
+		if (rtwn_rx_check_pre_alloc(sc, stat) == 0) {
+			data->m = m_getm(NULL, totlen, M_NOWAIT, MT_DATA);
+			if (data->m != NULL) {
+				m_copyback(data->m, 0, uc->uc_rx_stat_len,
+				    (caddr_t)stat);
+
+				if (rtwn_check_frame(sc, data->m) != 0) {
+					m_freem(data->m);
+					data->m = NULL;
+					counter_u64_add(ic->ic_ierrors, 1);
+				}
+			} else
+				counter_u64_add(ic->ic_ierrors, 1);
+		} else
+			counter_u64_add(ic->ic_ierrors, 1);
+
+		uc->uc_rx_off = sizeof(*stat);
+	}
+
+	/* If mbuf allocation fails just discard the data. */
+	min_len = min(totlen - uc->uc_rx_off, len);
+	if (data->m != NULL)
+		m_copyback(data->m, uc->uc_rx_off, min_len, buf);
+
+	uc->uc_rx_off += min_len;
+	if (uc->uc_rx_off == totlen) {
+		/* Align next frame. */
+		min_len = rtwn_usb_align_rx(uc,
+		    orig_len - len + min_len, orig_len);
+		min_len -= (orig_len - len);
+		KASSERT(len >= min_len, ("%s: len (%d) < min_len (%d)!\n",
+		    __func__, len, min_len));
+
+		/* Clear mbuf stats. */
+		uc->uc_rx_stat_len = 0;
+		uc->uc_rx_off = 0;
+	}
+	len -= min_len;
+	buf += min_len;
+end:
+	if (uc->uc_rx_stat_len == 0)
+		return (rtwn_rxeof(sc, data, buf, len));
+	else
+		return (NULL);
+}
+
+static struct mbuf *
+rtwn_rxeof(struct rtwn_softc *sc, struct rtwn_data *data, uint8_t *buf,
+    int len)
 {
 	struct rtwn_usb_softc *uc = RTWN_USB_SOFTC(sc);
-	struct r92c_rx_stat *stat;
+	struct rtwn_rx_stat_common *stat;
 	struct mbuf *m, *m0 = NULL;
 	uint32_t rxdw0;
 	int totlen, pktlen, infosz;
 
+	/* Prepend defragmented frame (if any). */
+	if (data->m != NULL) {
+		m0 = m = data->m;
+		data->m = NULL;
+	}
+
 	/* Process packets. */
 	while (len >= sizeof(*stat)) {
-		stat = (struct r92c_rx_stat *)buf;
+		stat = (struct rtwn_rx_stat_common *)buf;
 		rxdw0 = le32toh(stat->rxdw0);
 
-		pktlen = MS(rxdw0, R92C_RXDW0_PKTLEN);
+		pktlen = MS(rxdw0, RTWN_RXDW0_PKTLEN);
 		if (__predict_false(pktlen == 0))
 			break;
 
-		infosz = MS(rxdw0, R92C_RXDW0_INFOSZ) * 8;
+		infosz = MS(rxdw0, RTWN_RXDW0_INFOSZ) * 8;
 
 		/* Make sure everything fits in xfer. */
 		totlen = sizeof(*stat) + infosz + pktlen;
 		if (totlen > len) {
-			device_printf(sc->sc_dev,
-			    "%s: totlen (%d) > len (%d)!\n",
+			RTWN_DPRINTF(sc, RTWN_DEBUG_RECV,
+			    "%s: frame is fragmented (totlen %d len %d)\n",
 			    __func__, totlen, len);
 			break;
 		}
@@ -170,9 +276,9 @@ rtwn_rxeof(struct rtwn_softc *sc, uint8_t *buf, int len)
 		if (m0 == NULL)
 			m0 = m = rtwn_rx_copy_to_mbuf(sc, stat, totlen);
 		else {
-			m->m_next = rtwn_rx_copy_to_mbuf(sc, stat, totlen);
-			if (m->m_next != NULL)
-				m = m->m_next;
+			m->m_nextpkt = rtwn_rx_copy_to_mbuf(sc, stat, totlen);
+			if (m->m_nextpkt != NULL)
+				m = m->m_nextpkt;
 		}
 
 		/* Align next frame. */
@@ -181,6 +287,9 @@ rtwn_rxeof(struct rtwn_softc *sc, uint8_t *buf, int len)
 		len -= totlen;
 	}
 
+	if (len > 0)
+		(void)rtwn_rxeof_fragmented(uc, data, buf, len);
+
 	return (m0);
 }
 
@@ -195,15 +304,19 @@ rtwn_report_intr(struct rtwn_usb_softc *uc, struct usb_xfer *xfer,
 
 	usbd_xfer_status(xfer, &len, NULL, NULL, NULL);
 
-	if (__predict_false(len < sizeof(struct r92c_rx_stat))) {
+	if (__predict_false(len < sizeof(struct rtwn_rx_stat_common) &&
+	    uc->uc_rx_stat_len == 0)) {
 		counter_u64_add(ic->ic_ierrors, 1);
 		return (NULL);
 	}
 
 	buf = data->buf;
+	if (uc->uc_rx_stat_len > 0)
+		return (rtwn_rxeof_fragmented(uc, data, data->buf, len));
+
 	switch (rtwn_classify_intr(sc, buf, len)) {
 	case RTWN_RX_DATA:
-		return (rtwn_rxeof(sc, buf, len));
+		return (rtwn_rxeof(sc, data, buf, len));
 	case RTWN_RX_TX_REPORT:
 		if (sc->sc_ratectl != RTWN_RATECTL_NET80211) {
 			/* shouldn't happen */
@@ -240,11 +353,11 @@ rtwn_report_intr(struct rtwn_usb_softc *uc, struct usb_xfer *xfer,
 static struct ieee80211_node *
 rtwn_rx_frame(struct rtwn_softc *sc, struct mbuf *m)
 {
-	struct r92c_rx_stat stat;
+	struct rtwn_rx_stat_common stat;
 
 	/* Imitate PCIe layout. */
-	m_copydata(m, 0, sizeof(struct r92c_rx_stat), (caddr_t)&stat);
-	m_adj(m, sizeof(struct r92c_rx_stat));
+	m_copydata(m, 0, sizeof(stat), (caddr_t)&stat);
+	m_adj(m, sizeof(stat));
 
 	return (rtwn_rx_common(sc, m, &stat));
 }
@@ -289,8 +402,8 @@ tr_setup:
 		 * callback and safe to unlock.
 		 */
 		while (m != NULL) {
-			next = m->m_next;
-			m->m_next = NULL;
+			next = m->m_nextpkt;
+			m->m_nextpkt = NULL;
 
 			ni = rtwn_rx_frame(sc, m);
 
@@ -314,6 +427,8 @@ tr_setup:
 			STAILQ_INSERT_TAIL(&uc->uc_rx_inactive, data, next);
 		}
 		if (error != USB_ERR_CANCELLED) {
+			/* XXX restart device if frame was fragmented? */
+
 			usbd_xfer_set_stall(xfer);
 			counter_u64_add(ic->ic_ierrors, 1);
 			goto tr_setup;
diff --git a/freebsd/sys/dev/rtwn/usb/rtwn_usb_tx.c b/freebsd/sys/dev/rtwn/usb/rtwn_usb_tx.c
index 7bede4dc..61f0ba43 100644
--- a/freebsd/sys/dev/rtwn/usb/rtwn_usb_tx.c
+++ b/freebsd/sys/dev/rtwn/usb/rtwn_usb_tx.c
@@ -235,6 +235,9 @@ rtwn_usb_tx_start(struct rtwn_softc *sc, struct ieee80211_node *ni,
 
 	RTWN_ASSERT_LOCKED(sc);
 
+	if (m->m_pkthdr.len + sc->txdesc_len > RTWN_USB_TXBUFSZ)
+		return (EINVAL);
+
 	data = rtwn_usb_getbuf(uc);
 	if (data == NULL)
 		return (ENOBUFS);
diff --git a/freebsd/sys/dev/rtwn/usb/rtwn_usb_var.h b/freebsd/sys/dev/rtwn/usb/rtwn_usb_var.h
index be7f8f19..7ef21463 100644
--- a/freebsd/sys/dev/rtwn/usb/rtwn_usb_var.h
+++ b/freebsd/sys/dev/rtwn/usb/rtwn_usb_var.h
@@ -21,6 +21,14 @@
 #ifndef RTWN_USBVAR_H
 #define RTWN_USBVAR_H
 
+#include <dev/rtwn/if_rtwnreg.h>	/* for struct rtwn_rx_stat_common */
+
+#define RTWN_USB_RXBUFSZ_UNIT		(512)
+#define RTWN_USB_RXBUFSZ_MIN		( 4)
+#define RTWN_USB_RXBUFSZ_DEF		(24)
+#define RTWN_USB_RXBUFSZ_MAX		(64)
+#define RTWN_USB_TXBUFSZ		(16 * 1024)
+
 #define RTWN_IFACE_INDEX		0
 
 #define RTWN_USB_RX_LIST_COUNT		1
@@ -56,6 +64,12 @@ struct rtwn_usb_softc {
 	struct rtwn_data	uc_rx[RTWN_USB_RX_LIST_COUNT];
 	rtwn_datahead		uc_rx_active;
 	rtwn_datahead		uc_rx_inactive;
+	int			uc_rx_buf_size;
+
+	struct rtwn_rx_stat_common uc_rx_stat;
+	int			uc_rx_stat_len;
+	int			uc_rx_off;
+
 	struct rtwn_data	uc_tx[RTWN_USB_TX_LIST_COUNT];
 	rtwn_datahead		uc_tx_active;
 	rtwn_datahead		uc_tx_inactive;
diff --git a/freebsd/sys/dev/sdhci/sdhci.c b/freebsd/sys/dev/sdhci/sdhci.c
index c87199a8..f1616a6e 100644
--- a/freebsd/sys/dev/sdhci/sdhci.c
+++ b/freebsd/sys/dev/sdhci/sdhci.c
@@ -2,6 +2,7 @@
 
 /*-
  * Copyright (c) 2008 Alexander Motin <mav@FreeBSD.org>
+ * Copyright (c) 2017 Marius Strobl <marius@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -34,7 +35,9 @@ __FBSDID("$FreeBSD$");
 #include <sys/callout.h>
 #include <sys/conf.h>
 #include <sys/kernel.h>
+#include <sys/kobj.h>
 #include <sys/lock.h>
+#include <sys/malloc.h>
 #include <sys/module.h>
 #include <sys/mutex.h>
 #include <rtems/bsd/sys/resource.h>
@@ -50,13 +53,22 @@ __FBSDID("$FreeBSD$");
 #include <dev/mmc/mmcreg.h>
 #include <dev/mmc/mmcbrvar.h>
 
+#include <dev/sdhci/sdhci.h>
+
+#include <cam/cam.h>
+#include <cam/cam_ccb.h>
+#include <cam/cam_debug.h>
+#include <cam/cam_sim.h>
+#include <cam/cam_xpt_sim.h>
+
 #include <rtems/bsd/local/mmcbr_if.h>
-#include "sdhci.h"
 #include <rtems/bsd/local/sdhci_if.h>
 
+#include <rtems/bsd/local/opt_mmccam.h>
+
 SYSCTL_NODE(_hw, OID_AUTO, sdhci, CTLFLAG_RD, 0, "sdhci driver");
 
-static int sdhci_debug;
+static int sdhci_debug = 0;
 SYSCTL_INT(_hw_sdhci, OID_AUTO, debug, CTLFLAG_RWTUN, &sdhci_debug, 0,
     "Debug level");
 u_int sdhci_quirk_clear = 0;
@@ -78,17 +90,30 @@ SYSCTL_INT(_hw_sdhci, OID_AUTO, quirk_set, CTLFLAG_RWTUN, &sdhci_quirk_set, 0,
 #define	WR_MULTI_4(slot, off, ptr, count)	\
     SDHCI_WRITE_MULTI_4((slot)->bus, (slot), (off), (ptr), (count))
 
+static void sdhci_card_poll(void *arg);
+static void sdhci_card_task(void *arg, int pending);
+static int sdhci_exec_tuning(struct sdhci_slot *slot, bool reset);
+static void sdhci_req_wakeup(struct mmc_request *req);
+static void sdhci_retune(void *arg);
 static void sdhci_set_clock(struct sdhci_slot *slot, uint32_t clock);
 static void sdhci_start(struct sdhci_slot *slot);
 static void sdhci_start_data(struct sdhci_slot *slot, struct mmc_data *data);
 
-static void sdhci_card_poll(void *);
-static void sdhci_card_task(void *, int);
+#ifdef MMCCAM
+/* CAM-related */
+int sdhci_cam_get_possible_host_clock(struct sdhci_slot *slot, int proposed_clock);
+static int sdhci_cam_update_ios(struct sdhci_slot *slot);
+static int sdhci_cam_request(struct sdhci_slot *slot, union ccb *ccb);
+static void sdhci_cam_action(struct cam_sim *sim, union ccb *ccb);
+static void sdhci_cam_poll(struct cam_sim *sim);
+static int sdhci_cam_settran_settings(struct sdhci_slot *slot, union ccb *ccb);
+#endif
 
 /* helper routines */
 static void sdhci_dumpregs(struct sdhci_slot *slot);
 static int slot_printf(struct sdhci_slot *slot, const char * fmt, ...)
     __printflike(2, 3);
+static uint32_t sdhci_tuning_intmask(struct sdhci_slot *slot);
 
 #define	SDHCI_LOCK(_slot)		mtx_lock(&(_slot)->mtx)
 #define	SDHCI_UNLOCK(_slot)		mtx_unlock(&(_slot)->mtx)
@@ -169,13 +194,13 @@ sdhci_dumpregs(struct sdhci_slot *slot)
 	    RD1(slot, SDHCI_TIMEOUT_CONTROL), RD4(slot, SDHCI_INT_STATUS));
 	slot_printf(slot, "Int enab: 0x%08x | Sig enab: 0x%08x\n",
 	    RD4(slot, SDHCI_INT_ENABLE), RD4(slot, SDHCI_SIGNAL_ENABLE));
-	slot_printf(slot, "AC12 err: 0x%08x | Host ctl2: 0x%08x\n",
+	slot_printf(slot, "AC12 err: 0x%08x | Host ctl2:0x%08x\n",
 	    RD2(slot, SDHCI_ACMD12_ERR), RD2(slot, SDHCI_HOST_CONTROL2));
 	slot_printf(slot, "Caps:     0x%08x | Caps2:    0x%08x\n",
 	    RD4(slot, SDHCI_CAPABILITIES), RD4(slot, SDHCI_CAPABILITIES2));
 	slot_printf(slot, "Max curr: 0x%08x | ADMA err: 0x%08x\n",
 	    RD4(slot, SDHCI_MAX_CURRENT), RD1(slot, SDHCI_ADMA_ERR));
-	slot_printf(slot, "ADMA addr: 0x%08x | Slot int: 0x%08x\n",
+	slot_printf(slot, "ADMA addr:0x%08x | Slot int: 0x%08x\n",
 	    RD4(slot, SDHCI_ADMA_ADDRESS_LO), RD2(slot, SDHCI_SLOT_INT_STATUS));
 
 	slot_printf(slot,
@@ -242,6 +267,21 @@ sdhci_reset(struct sdhci_slot *slot, uint8_t mask)
 	}
 }
 
+static uint32_t
+sdhci_tuning_intmask(struct sdhci_slot *slot)
+{
+	uint32_t intmask;
+
+	intmask = 0;
+	if (slot->opt & SDHCI_TUNING_SUPPORTED) {
+		intmask |= SDHCI_INT_TUNEERR;
+		if (slot->retune_mode == SDHCI_RETUNE_MODE_2 ||
+		    slot->retune_mode == SDHCI_RETUNE_MODE_3)
+			intmask |= SDHCI_INT_RETUNE;
+	}
+	return (intmask);
+}
+
 static void
 sdhci_init(struct sdhci_slot *slot)
 {
@@ -261,7 +301,7 @@ sdhci_init(struct sdhci_slot *slot)
 		slot->intmask |= SDHCI_INT_CARD_REMOVE | SDHCI_INT_CARD_INSERT;
 	}
 
-	WR4(slot, SDHCI_INT_ENABLE, slot->intmask);
+	WR4(slot, SDHCI_INT_ENABLE, slot->intmask | sdhci_tuning_intmask(slot));
 	WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask);
 }
 
@@ -368,6 +408,7 @@ sdhci_set_clock(struct sdhci_slot *slot, uint32_t clock)
 static void
 sdhci_set_power(struct sdhci_slot *slot, u_char power)
 {
+	int i;
 	uint8_t pwr;
 
 	if (slot->power == power)
@@ -396,9 +437,20 @@ sdhci_set_power(struct sdhci_slot *slot, u_char power)
 		break;
 	}
 	WR1(slot, SDHCI_POWER_CONTROL, pwr);
-	/* Turn on the power. */
+	/*
+	 * Turn on VDD1 power.  Note that at least some Intel controllers can
+	 * fail to enable bus power on the first try after transiting from D3
+	 * to D0, so we give them up to 2 ms.
+	 */
 	pwr |= SDHCI_POWER_ON;
-	WR1(slot, SDHCI_POWER_CONTROL, pwr);
+	for (i = 0; i < 20; i++) {
+		WR1(slot, SDHCI_POWER_CONTROL, pwr);
+		if (RD1(slot, SDHCI_POWER_CONTROL) & SDHCI_POWER_ON)
+			break;
+		DELAY(100);
+	}
+	if (!(RD1(slot, SDHCI_POWER_CONTROL) & SDHCI_POWER_ON))
+		slot_printf(slot, "Bus power failed to enable");
 
 	if (slot->quirks & SDHCI_QUIRK_INTEL_POWER_UP_RESET) {
 		WR1(slot, SDHCI_POWER_CONTROL, pwr | 0x10);
@@ -521,25 +573,93 @@ sdhci_card_task(void *arg, int pending __unused)
 
 	SDHCI_LOCK(slot);
 	if (SDHCI_GET_CARD_PRESENT(slot->bus, slot)) {
+#ifdef MMCCAM
+		if (slot->card_present == 0) {
+#else
 		if (slot->dev == NULL) {
+#endif
 			/* If card is present - attach mmc bus. */
 			if (bootverbose || sdhci_debug)
 				slot_printf(slot, "Card inserted\n");
-			slot->dev = device_add_child(slot->bus, "mmc", -1);
-			device_set_ivars(slot->dev, slot);
+#ifdef MMCCAM
+			slot->card_present = 1;
+			union ccb *ccb;
+			uint32_t pathid;
+			pathid = cam_sim_path(slot->sim);
+			ccb = xpt_alloc_ccb_nowait();
+			if (ccb == NULL) {
+				slot_printf(slot, "Unable to alloc CCB for rescan\n");
+				SDHCI_UNLOCK(slot);
+				return;
+			}
+
+			/*
+			 * We create a rescan request for BUS:0:0, since the card
+			 * will be at lun 0.
+			 */
+			if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid,
+					    /* target */ 0, /* lun */ 0) != CAM_REQ_CMP) {
+				slot_printf(slot, "Unable to create path for rescan\n");
+				SDHCI_UNLOCK(slot);
+				xpt_free_ccb(ccb);
+				return;
+			}
+			SDHCI_UNLOCK(slot);
+			xpt_rescan(ccb);
+#else
+			d = slot->dev = device_add_child(slot->bus, "mmc", -1);
 			SDHCI_UNLOCK(slot);
-			device_probe_and_attach(slot->dev);
+			if (d) {
+				device_set_ivars(d, slot);
+				(void)device_probe_and_attach(d);
+			}
+#endif
 		} else
 			SDHCI_UNLOCK(slot);
 	} else {
+#ifdef MMCCAM
+		if (slot->card_present == 1) {
+#else
 		if (slot->dev != NULL) {
+#endif
 			/* If no card present - detach mmc bus. */
 			if (bootverbose || sdhci_debug)
 				slot_printf(slot, "Card removed\n");
 			d = slot->dev;
 			slot->dev = NULL;
+#ifdef MMCCAM
+			slot->card_present = 0;
+			union ccb *ccb;
+			uint32_t pathid;
+			pathid = cam_sim_path(slot->sim);
+			ccb = xpt_alloc_ccb_nowait();
+			if (ccb == NULL) {
+				slot_printf(slot, "Unable to alloc CCB for rescan\n");
+				SDHCI_UNLOCK(slot);
+				return;
+			}
+
+			/*
+			 * We create a rescan request for BUS:0:0, since the card
+			 * will be at lun 0.
+			 */
+			if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid,
+					    /* target */ 0, /* lun */ 0) != CAM_REQ_CMP) {
+				slot_printf(slot, "Unable to create path for rescan\n");
+				SDHCI_UNLOCK(slot);
+				xpt_free_ccb(ccb);
+				return;
+			}
 			SDHCI_UNLOCK(slot);
+			xpt_rescan(ccb);
+#else
+			slot->intmask &= ~sdhci_tuning_intmask(slot);
+			WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask);
+			slot->opt &= ~SDHCI_TUNING_ENABLED;
+			SDHCI_UNLOCK(slot);
+			callout_drain(&slot->retune_callout);
 			device_delete_child(slot->bus, d);
+#endif
 		} else
 			SDHCI_UNLOCK(slot);
 	}
@@ -561,7 +681,11 @@ sdhci_handle_card_present_locked(struct sdhci_slot *slot, bool is_present)
 	 * because once power is removed, a full card re-init is needed, and
 	 * that happens by deleting and recreating the child device.
 	 */
+#ifdef MMCCAM
+	was_present = slot->card_present;
+#else
 	was_present = slot->dev != NULL;
+#endif
 	if (!was_present && is_present) {
 		taskqueue_enqueue_timeout(taskqueue_swi_giant,
 		    &slot->card_delayed_task, -SDHCI_INSERT_DELAY_TICKS);
@@ -593,10 +717,13 @@ sdhci_card_poll(void *arg)
 int
 sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
 {
+	kobjop_desc_t kobj_desc;
+	kobj_method_t *kobj_method;
 	uint32_t caps, caps2, freq, host_caps;
 	int err;
 
 	SDHCI_LOCK_INIT(slot);
+
 	slot->num = num;
 	slot->bus = dev;
 
@@ -617,6 +744,7 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
 	    BUS_DMA_NOWAIT, &slot->dmamap);
 	if (err != 0) {
 		device_printf(dev, "Can't alloc DMA memory\n");
+		bus_dma_tag_destroy(slot->dmatag);
 		SDHCI_LOCK_DESTROY(slot);
 		return (err);
 	}
@@ -626,6 +754,8 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
 	    sdhci_getaddr, &slot->paddr, 0);
 	if (err != 0 || slot->paddr == 0) {
 		device_printf(dev, "Can't load DMA memory\n");
+		bus_dmamem_free(slot->dmatag, slot->dmamem, slot->dmamap);
+		bus_dma_tag_destroy(slot->dmatag);
 		SDHCI_LOCK_DESTROY(slot);
 		if (err)
 			return (err);
@@ -633,8 +763,6 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
 			return (EFAULT);
 	}
 
-	/* Initialize slot. */
-	sdhci_init(slot);
 	slot->version = (RD2(slot, SDHCI_HOST_VERSION)
 		>> SDHCI_SPEC_VER_SHIFT) & SDHCI_SPEC_VER_MASK;
 	if (slot->quirks & SDHCI_QUIRK_MISSING_CAPS) {
@@ -647,6 +775,22 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
 		else
 			caps2 = 0;
 	}
+	if (slot->version >= SDHCI_SPEC_300) {
+		if ((caps & SDHCI_SLOTTYPE_MASK) != SDHCI_SLOTTYPE_REMOVABLE &&
+		    (caps & SDHCI_SLOTTYPE_MASK) != SDHCI_SLOTTYPE_EMBEDDED) {
+			device_printf(dev,
+			    "Driver doesn't support shared bus slots\n");
+			bus_dmamap_unload(slot->dmatag, slot->dmamap);
+			bus_dmamem_free(slot->dmatag, slot->dmamem,
+			    slot->dmamap);
+			bus_dma_tag_destroy(slot->dmatag);
+			SDHCI_LOCK_DESTROY(slot);
+			return (ENXIO);
+		} else if ((caps & SDHCI_SLOTTYPE_MASK) ==
+		    SDHCI_SLOTTYPE_EMBEDDED) {
+			slot->opt |= SDHCI_SLOT_EMBEDDED | SDHCI_NON_REMOVABLE;
+		}
+	}
 	/* Calculate base clock frequency. */
 	if (slot->version >= SDHCI_SPEC_300)
 		freq = (caps & SDHCI_CLOCK_V3_BASE_MASK) >>
@@ -696,12 +840,14 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
 	    slot->host.host_ocr |= MMC_OCR_320_330 | MMC_OCR_330_340;
 	if (caps & SDHCI_CAN_VDD_300)
 	    slot->host.host_ocr |= MMC_OCR_290_300 | MMC_OCR_300_310;
-	if (caps & SDHCI_CAN_VDD_180)
+	/* 1.8V VDD is not supposed to be used for removable cards. */
+	if ((caps & SDHCI_CAN_VDD_180) && (slot->opt & SDHCI_SLOT_EMBEDDED))
 	    slot->host.host_ocr |= MMC_OCR_LOW_VOLTAGE;
 	if (slot->host.host_ocr == 0) {
 		device_printf(dev, "Hardware doesn't report any "
 		    "support voltages.\n");
 	}
+
 	host_caps = MMC_CAP_4_BIT_DATA;
 	if (caps & SDHCI_CAN_DO_8BITBUS)
 		host_caps |= MMC_CAP_8_BIT_DATA;
@@ -711,6 +857,8 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
 		host_caps |= MMC_CAP_BOOT_NOACC;
 	if (slot->quirks & SDHCI_QUIRK_WAIT_WHILE_BUSY)
 		host_caps |= MMC_CAP_WAIT_WHILE_BUSY;
+
+	/* Determine supported UHS-I and eMMC modes. */
 	if (caps2 & (SDHCI_CAN_SDR50 | SDHCI_CAN_SDR104 | SDHCI_CAN_DDR50))
 		host_caps |= MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25;
 	if (caps2 & SDHCI_CAN_SDR104) {
@@ -727,12 +875,91 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
 	if (slot->quirks & SDHCI_QUIRK_CAPS_BIT63_FOR_MMC_HS400 &&
 	    caps2 & SDHCI_CAN_MMC_HS400)
 		host_caps |= MMC_CAP_MMC_HS400;
+
+	/*
+	 * Disable UHS-I and eMMC modes if the set_uhs_timing method is the
+	 * default NULL implementation.
+	 */
+	kobj_desc = &sdhci_set_uhs_timing_desc;
+	kobj_method = kobj_lookup_method(((kobj_t)dev)->ops->cls, NULL,
+	    kobj_desc);
+	if (kobj_method == &kobj_desc->deflt)
+		host_caps &= ~(MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25 |
+		    MMC_CAP_UHS_SDR50 | MMC_CAP_UHS_DDR50 | MMC_CAP_UHS_SDR104 |
+		    MMC_CAP_MMC_DDR52 | MMC_CAP_MMC_HS200 | MMC_CAP_MMC_HS400);
+
+#define	SDHCI_CAP_MODES_TUNING(caps2)					\
+    (((caps2) & SDHCI_TUNE_SDR50 ? MMC_CAP_UHS_SDR50 : 0) |		\
+    MMC_CAP_UHS_DDR50 | MMC_CAP_UHS_SDR104 | MMC_CAP_MMC_HS200 |	\
+    MMC_CAP_MMC_HS400)
+
+	/*
+	 * Disable UHS-I and eMMC modes that require (re-)tuning if either
+	 * the tune or re-tune method is the default NULL implementation.
+	 */
+	kobj_desc = &mmcbr_tune_desc;
+	kobj_method = kobj_lookup_method(((kobj_t)dev)->ops->cls, NULL,
+	    kobj_desc);
+	if (kobj_method == &kobj_desc->deflt)
+		goto no_tuning;
+	kobj_desc = &mmcbr_retune_desc;
+	kobj_method = kobj_lookup_method(((kobj_t)dev)->ops->cls, NULL,
+	    kobj_desc);
+	if (kobj_method == &kobj_desc->deflt) {
+no_tuning:
+		host_caps &= ~(SDHCI_CAP_MODES_TUNING(caps2));
+	}
+
+	/* Allocate tuning structures and determine tuning parameters. */
+	if (host_caps & SDHCI_CAP_MODES_TUNING(caps2)) {
+		slot->opt |= SDHCI_TUNING_SUPPORTED;
+		slot->tune_req = malloc(sizeof(*slot->tune_req), M_DEVBUF,
+		    M_WAITOK);
+		slot->tune_cmd = malloc(sizeof(*slot->tune_cmd), M_DEVBUF,
+		    M_WAITOK);
+		slot->tune_data = malloc(sizeof(*slot->tune_data), M_DEVBUF,
+		    M_WAITOK);
+		if (caps2 & SDHCI_TUNE_SDR50)
+			slot->opt |= SDHCI_SDR50_NEEDS_TUNING;
+		slot->retune_mode = (caps2 & SDHCI_RETUNE_MODES_MASK) >>
+		    SDHCI_RETUNE_MODES_SHIFT;
+		if (slot->retune_mode == SDHCI_RETUNE_MODE_1) {
+			slot->retune_count = (caps2 & SDHCI_RETUNE_CNT_MASK) >>
+			    SDHCI_RETUNE_CNT_SHIFT;
+			if (slot->retune_count > 0xb) {
+				device_printf(dev, "Unknown re-tuning count "
+				    "%x, using 1 sec\n", slot->retune_count);
+				slot->retune_count = 1;
+			} else if (slot->retune_count != 0)
+				slot->retune_count =
+				    1 << (slot->retune_count - 1);
+		}
+	}
+
+#undef SDHCI_CAP_MODES_TUNING
+
+	/* Determine supported VCCQ signaling levels. */
 	host_caps |= MMC_CAP_SIGNALING_330;
 	if (host_caps & (MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25 |
-	    MMC_CAP_UHS_SDR50 | MMC_CAP_UHS_SDR104 | MMC_CAP_UHS_DDR50 |
+	    MMC_CAP_UHS_SDR50 | MMC_CAP_UHS_DDR50 | MMC_CAP_UHS_SDR104 |
 	    MMC_CAP_MMC_DDR52_180 | MMC_CAP_MMC_HS200_180 |
 	    MMC_CAP_MMC_HS400_180))
-		host_caps |= MMC_CAP_SIGNALING_180;
+		host_caps |= MMC_CAP_SIGNALING_120 | MMC_CAP_SIGNALING_180;
+
+	/*
+	 * Disable 1.2 V and 1.8 V signaling if the switch_vccq method is the
+	 * default NULL implementation.  Disable 1.2 V support if it's the
+	 * generic SDHCI implementation.
+	 */
+	kobj_desc = &mmcbr_switch_vccq_desc;
+	kobj_method = kobj_lookup_method(((kobj_t)dev)->ops->cls, NULL,
+	    kobj_desc);
+	if (kobj_method == &kobj_desc->deflt)
+		host_caps &= ~(MMC_CAP_SIGNALING_120 | MMC_CAP_SIGNALING_180);
+	else if (kobj_method->func == (kobjop_t)sdhci_generic_switch_vccq)
+		host_caps &= ~MMC_CAP_SIGNALING_120;
+
+	/* Determine supported driver types (type B is always mandatory). */
 	if (caps2 & SDHCI_CAN_DRIVE_TYPE_A)
 		host_caps |= MMC_CAP_DRIVER_TYPE_A;
 	if (caps2 & SDHCI_CAN_DRIVE_TYPE_C)
@@ -761,20 +988,24 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
 
 	if (bootverbose || sdhci_debug) {
 		slot_printf(slot,
-		    "%uMHz%s %s VDD:%s%s%s VCCQ: 3.3V%s%s DRV: B%s%s%s %s\n",
+		    "%uMHz%s %s VDD:%s%s%s VCCQ: 3.3V%s%s DRV: B%s%s%s %s %s\n",
 		    slot->max_clk / 1000000,
 		    (caps & SDHCI_CAN_DO_HISPD) ? " HS" : "",
 		    (host_caps & MMC_CAP_8_BIT_DATA) ? "8bits" :
 			((host_caps & MMC_CAP_4_BIT_DATA) ? "4bits" : "1bit"),
 		    (caps & SDHCI_CAN_VDD_330) ? " 3.3V" : "",
 		    (caps & SDHCI_CAN_VDD_300) ? " 3.0V" : "",
-		    (caps & SDHCI_CAN_VDD_180) ? " 1.8V" : "",
+		    ((caps & SDHCI_CAN_VDD_180) &&
+		    (slot->opt & SDHCI_SLOT_EMBEDDED)) ? " 1.8V" : "",
 		    (host_caps & MMC_CAP_SIGNALING_180) ? " 1.8V" : "",
 		    (host_caps & MMC_CAP_SIGNALING_120) ? " 1.2V" : "",
-		    (caps2 & SDHCI_CAN_DRIVE_TYPE_A) ? "A" : "",
-		    (caps2 & SDHCI_CAN_DRIVE_TYPE_C) ? "C" : "",
-		    (caps2 & SDHCI_CAN_DRIVE_TYPE_D) ? "D" : "",
-		    (slot->opt & SDHCI_HAVE_DMA) ? "DMA" : "PIO");
+		    (host_caps & MMC_CAP_DRIVER_TYPE_A) ? "A" : "",
+		    (host_caps & MMC_CAP_DRIVER_TYPE_C) ? "C" : "",
+		    (host_caps & MMC_CAP_DRIVER_TYPE_D) ? "D" : "",
+		    (slot->opt & SDHCI_HAVE_DMA) ? "DMA" : "PIO",
+		    (slot->opt & SDHCI_SLOT_EMBEDDED) ? "embedded" :
+		    (slot->opt & SDHCI_NON_REMOVABLE) ? "non-removable" :
+		    "removable");
 		if (host_caps & (MMC_CAP_MMC_DDR52 | MMC_CAP_MMC_HS200 |
 		    MMC_CAP_MMC_HS400 | MMC_CAP_MMC_ENH_STROBE))
 			slot_printf(slot, "eMMC:%s%s%s%s\n",
@@ -793,6 +1024,9 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
 			    (host_caps & MMC_CAP_UHS_SDR50) ? " SDR50" : "",
 			    (host_caps & MMC_CAP_UHS_SDR104) ? " SDR104" : "",
 			    (host_caps & MMC_CAP_UHS_DDR50) ? " DDR50" : "");
+		if (slot->opt & SDHCI_TUNING_SUPPORTED)
+			slot_printf(slot, "Re-tuning count %d secs, mode %d\n",
+			    slot->retune_count, slot->retune_mode + 1);
 		sdhci_dumpregs(slot);
 	}
 
@@ -806,6 +1040,7 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
 		sdhci_card_task, slot);
 	callout_init(&slot->card_poll_callout, 1);
 	callout_init_mtx(&slot->timeout_callout, &slot->mtx, 0);
+	callout_init_mtx(&slot->retune_callout, &slot->mtx, 0);
 
 	if ((slot->quirks & SDHCI_QUIRK_POLL_CARD_PRESENT) &&
 	    !(slot->opt & SDHCI_NON_REMOVABLE)) {
@@ -813,6 +1048,8 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
 		    SDHCI_CARD_PRESENT_TICKS, sdhci_card_poll, slot);
 	}
 
+	sdhci_init(slot);
+
 	return (0);
 }
 
@@ -830,6 +1067,7 @@ sdhci_cleanup_slot(struct sdhci_slot *slot)
 
 	callout_drain(&slot->timeout_callout);
 	callout_drain(&slot->card_poll_callout);
+	callout_drain(&slot->retune_callout);
 	taskqueue_drain(taskqueue_swi_giant, &slot->card_task);
 	taskqueue_drain_timeout(taskqueue_swi_giant, &slot->card_delayed_task);
 
@@ -846,6 +1084,11 @@ sdhci_cleanup_slot(struct sdhci_slot *slot)
 	bus_dmamap_unload(slot->dmatag, slot->dmamap);
 	bus_dmamem_free(slot->dmatag, slot->dmamem, slot->dmamap);
 	bus_dma_tag_destroy(slot->dmatag);
+	if (slot->opt & SDHCI_TUNING_SUPPORTED) {
+		free(slot->tune_req, M_DEVBUF);
+		free(slot->tune_cmd, M_DEVBUF);
+		free(slot->tune_data, M_DEVBUF);
+	}
 
 	SDHCI_LOCK_DESTROY(slot);
 
@@ -856,7 +1099,16 @@ int
 sdhci_generic_suspend(struct sdhci_slot *slot)
 {
 
+	/*
+	 * We expect the MMC layer to issue initial tuning after resume.
+	 * Otherwise, we'd need to indicate re-tuning including circuit reset
+	 * being required at least for re-tuning modes 1 and 2 ourselves.
+	 */
+	callout_drain(&slot->retune_callout);
+	SDHCI_LOCK(slot);
+	slot->opt &= ~SDHCI_TUNING_ENABLED;
 	sdhci_reset(slot, SDHCI_RESET_ALL);
+	SDHCI_UNLOCK(slot);
 
 	return (0);
 }
@@ -865,7 +1117,9 @@ int
 sdhci_generic_resume(struct sdhci_slot *slot)
 {
 
+	SDHCI_LOCK(slot);
 	sdhci_init(slot);
+	SDHCI_UNLOCK(slot);
 
 	return (0);
 }
@@ -899,15 +1153,18 @@ sdhci_generic_set_uhs_timing(device_t brdev __unused, struct sdhci_slot *slot)
 	if (slot->version < SDHCI_SPEC_300)
 		return;
 
+	SDHCI_ASSERT_LOCKED(slot);
 	ios = &slot->host.ios;
 	sdhci_set_clock(slot, 0);
 	hostctrl2 = RD2(slot, SDHCI_HOST_CONTROL2);
 	hostctrl2 &= ~SDHCI_CTRL2_UHS_MASK;
-	if (ios->timing == bus_timing_mmc_hs400 ||
-	    ios->timing == bus_timing_mmc_hs400es)
-		hostctrl2 |= SDHCI_CTRL2_MMC_HS400;
-	else if (ios->clock > SD_SDR50_MAX)
-		hostctrl2 |= SDHCI_CTRL2_UHS_SDR104;
+	if (ios->clock > SD_SDR50_MAX) {
+		if (ios->timing == bus_timing_mmc_hs400 ||
+		    ios->timing == bus_timing_mmc_hs400es)
+			hostctrl2 |= SDHCI_CTRL2_MMC_HS400;
+		else
+			hostctrl2 |= SDHCI_CTRL2_UHS_SDR104;
+	}
 	else if (ios->clock > SD_SDR25_MAX)
 		hostctrl2 |= SDHCI_CTRL2_UHS_SDR50;
 	else if (ios->clock > SD_SDR12_MAX) {
@@ -1019,6 +1276,222 @@ done:
 	return (err);
 }
 
+int
+sdhci_generic_tune(device_t brdev __unused, device_t reqdev, bool hs400)
+{
+	struct sdhci_slot *slot = device_get_ivars(reqdev);
+	struct mmc_ios *ios = &slot->host.ios;
+	struct mmc_command *tune_cmd;
+	struct mmc_data *tune_data;
+	uint32_t opcode;
+	int err;
+
+	if (!(slot->opt & SDHCI_TUNING_SUPPORTED))
+		return (0);
+
+	slot->retune_ticks = slot->retune_count * hz;
+	opcode = MMC_SEND_TUNING_BLOCK;
+	SDHCI_LOCK(slot);
+	switch (ios->timing) {
+	case bus_timing_mmc_hs400:
+		slot_printf(slot, "HS400 must be tuned in HS200 mode\n");
+		SDHCI_UNLOCK(slot);
+		return (EINVAL);
+	case bus_timing_mmc_hs200:
+		/*
+		 * In HS400 mode, controllers use the data strobe line to
+		 * latch data from the devices so periodic re-tuning isn't
+		 * expected to be required.
+		 */
+		if (hs400)
+			slot->retune_ticks = 0;
+		opcode = MMC_SEND_TUNING_BLOCK_HS200;
+		break;
+	case bus_timing_uhs_ddr50:
+	case bus_timing_uhs_sdr104:
+		break;
+	case bus_timing_uhs_sdr50:
+		if (slot->opt & SDHCI_SDR50_NEEDS_TUNING)
+			break;
+		/* FALLTHROUGH */
+	default:
+		SDHCI_UNLOCK(slot);
+		return (0);
+	}
+
+	tune_cmd = slot->tune_cmd;
+	memset(tune_cmd, 0, sizeof(*tune_cmd));
+	tune_cmd->opcode = opcode;
+	tune_cmd->flags = MMC_RSP_R1 | MMC_CMD_ADTC;
+	tune_data = tune_cmd->data = slot->tune_data;
+	memset(tune_data, 0, sizeof(*tune_data));
+	tune_data->len = (opcode == MMC_SEND_TUNING_BLOCK_HS200 &&
+	    ios->bus_width == bus_width_8) ? MMC_TUNING_LEN_HS200 :
+	    MMC_TUNING_LEN;
+	tune_data->flags = MMC_DATA_READ;
+	tune_data->mrq = tune_cmd->mrq = slot->tune_req;
+
+	slot->opt &= ~SDHCI_TUNING_ENABLED;
+	err = sdhci_exec_tuning(slot, true);
+	if (err == 0) {
+		slot->opt |= SDHCI_TUNING_ENABLED;
+		slot->intmask |= sdhci_tuning_intmask(slot);
+		WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask);
+		if (slot->retune_ticks) {
+			callout_reset(&slot->retune_callout, slot->retune_ticks,
+			    sdhci_retune, slot);
+		}
+	}
+	SDHCI_UNLOCK(slot);
+	return (err);
+}
+
+int
+sdhci_generic_retune(device_t brdev __unused, device_t reqdev, bool reset)
+{
+	struct sdhci_slot *slot = device_get_ivars(reqdev);
+	int err;
+
+	if (!(slot->opt & SDHCI_TUNING_ENABLED))
+		return (0);
+
+	/* HS400 must be tuned in HS200 mode. */
+	if (slot->host.ios.timing == bus_timing_mmc_hs400)
+		return (EINVAL);
+
+	SDHCI_LOCK(slot);
+	err = sdhci_exec_tuning(slot, reset);
+	/*
+	 * There are two ways sdhci_exec_tuning() can fail:
+	 * EBUSY should not actually happen when requests are only issued
+	 *	 with the host properly acquired, and
+	 * EIO   re-tuning failed (but it did work initially).
+	 *
+	 * In both cases, we should retry at later point if periodic re-tuning
+	 * is enabled.  Note that due to slot->retune_req not being cleared in
+	 * these failure cases, the MMC layer should trigger another attempt at
+	 * re-tuning with the next request anyway, though.
+	 */
+	if (slot->retune_ticks) {
+		callout_reset(&slot->retune_callout, slot->retune_ticks,
+		    sdhci_retune, slot);
+	}
+	SDHCI_UNLOCK(slot);
+	return (err);
+}
+
+static int
+sdhci_exec_tuning(struct sdhci_slot *slot, bool reset)
+{
+	struct mmc_request *tune_req;
+	struct mmc_command *tune_cmd;
+	int i;
+	uint32_t intmask;
+	uint16_t hostctrl2;
+	u_char opt;
+
+	SDHCI_ASSERT_LOCKED(slot);
+	if (slot->req != NULL)
+		return (EBUSY);
+
+	/* Tuning doesn't work with DMA enabled. */
+	opt = slot->opt;
+	slot->opt = opt & ~SDHCI_HAVE_DMA;
+
+	/*
+	 * Ensure that as documented, SDHCI_INT_DATA_AVAIL is the only
+	 * kind of interrupt we receive in response to a tuning request.
+	 */
+	intmask = slot->intmask;
+	slot->intmask = SDHCI_INT_DATA_AVAIL;
+	WR4(slot, SDHCI_SIGNAL_ENABLE, SDHCI_INT_DATA_AVAIL);
+
+	hostctrl2 = RD2(slot, SDHCI_HOST_CONTROL2);
+	if (reset)
+		hostctrl2 &= ~SDHCI_CTRL2_SAMPLING_CLOCK;
+	else
+		hostctrl2 |= SDHCI_CTRL2_SAMPLING_CLOCK;
+	WR2(slot, SDHCI_HOST_CONTROL2, hostctrl2 | SDHCI_CTRL2_EXEC_TUNING);
+
+	tune_req = slot->tune_req;
+	tune_cmd = slot->tune_cmd;
+	for (i = 0; i < MMC_TUNING_MAX; i++) {
+		memset(tune_req, 0, sizeof(*tune_req));
+#ifdef __rtems__
+		rtems_binary_semaphore_init(&tune_req->req_done,
+		    "sdhci_req_done");
+#endif /* __rtems__ */
+		tune_req->cmd = tune_cmd;
+		tune_req->done = sdhci_req_wakeup;
+		tune_req->done_data = slot;
+		slot->req = tune_req;
+		slot->flags = 0;
+		sdhci_start(slot);
+#ifndef __rtems__
+		while (!(tune_req->flags & MMC_REQ_DONE))
+			msleep(tune_req, &slot->mtx, 0, "sdhciet", 0);
+#else /* __rtems__ */
+		rtems_binary_semaphore_wait(&tune_req->req_done);
+		rtems_binary_semaphore_destroy(&tune_req->req_done);
+#endif /* __rtems__ */
+		if (!(tune_req->flags & MMC_TUNE_DONE))
+			break;
+		hostctrl2 = RD2(slot, SDHCI_HOST_CONTROL2);
+		if (!(hostctrl2 & SDHCI_CTRL2_EXEC_TUNING))
+			break;
+		if (tune_cmd->opcode == MMC_SEND_TUNING_BLOCK)
+			DELAY(1000);
+	}
+
+	slot->opt = opt;
+	slot->intmask = intmask;
+	WR4(slot, SDHCI_SIGNAL_ENABLE, intmask);
+
+	if ((hostctrl2 & (SDHCI_CTRL2_EXEC_TUNING |
+	    SDHCI_CTRL2_SAMPLING_CLOCK)) == SDHCI_CTRL2_SAMPLING_CLOCK) {
+		slot->retune_req = 0;
+		return (0);
+	}
+
+	slot_printf(slot, "Tuning failed, using fixed sampling clock\n");
+	WR2(slot, SDHCI_HOST_CONTROL2, hostctrl2 & ~(SDHCI_CTRL2_EXEC_TUNING |
+	    SDHCI_CTRL2_SAMPLING_CLOCK));
+	sdhci_reset(slot, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
+	return (EIO);
+}
+
+static void
+sdhci_retune(void *arg)
+{
+	struct sdhci_slot *slot = arg;
+
+	slot->retune_req |= SDHCI_RETUNE_REQ_NEEDED;
+}
+
+#ifdef MMCCAM
+static void
+sdhci_req_done(struct sdhci_slot *slot)
+{
+        union ccb *ccb;
+
+	if (__predict_false(sdhci_debug > 1))
+		slot_printf(slot, "%s\n", __func__);
+	if (slot->ccb != NULL && slot->curcmd != NULL) {
+		callout_stop(&slot->timeout_callout);
+                ccb = slot->ccb;
+                slot->ccb = NULL;
+		slot->curcmd = NULL;
+
+                /* Tell CAM the request is finished */
+                struct ccb_mmcio *mmcio;
+                mmcio = &ccb->mmcio;
+
+                ccb->ccb_h.status =
+                        (mmcio->cmd.error == 0 ? CAM_REQ_CMP : CAM_REQ_CMP_ERR);
+                xpt_done(ccb);
+	}
+}
+#else
 static void
 sdhci_req_done(struct sdhci_slot *slot)
 {
@@ -1032,6 +1505,21 @@ sdhci_req_done(struct sdhci_slot *slot)
 		req->done(req);
 	}
 }
+#endif
+
+static void
+sdhci_req_wakeup(struct mmc_request *req)
+{
+#ifndef __rtems__
+	struct sdhci_slot *slot;
+
+	slot = req->done_data;
+	req->flags |= MMC_REQ_DONE;
+	wakeup(req);
+#else /* __rtems__ */
+	rtems_binary_semaphore_post(&req->req_done);
+#endif /* __rtems__ */
+}
 
 static void
 sdhci_timeout(void *arg)
@@ -1039,13 +1527,13 @@ sdhci_timeout(void *arg)
 	struct sdhci_slot *slot = arg;
 
 	if (slot->curcmd != NULL) {
-		slot_printf(slot, " Controller timeout\n");
+		slot_printf(slot, "Controller timeout\n");
 		sdhci_dumpregs(slot);
 		sdhci_reset(slot, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
 		slot->curcmd->error = MMC_ERR_TIMEOUT;
 		sdhci_req_done(slot);
 	} else {
-		slot_printf(slot, " Spurious timeout - no active command\n");
+		slot_printf(slot, "Spurious timeout - no active command\n");
 	}
 }
 
@@ -1062,8 +1550,16 @@ sdhci_set_transfer_mode(struct sdhci_slot *slot, struct mmc_data *data)
 		mode |= SDHCI_TRNS_MULTI;
 	if (data->flags & MMC_DATA_READ)
 		mode |= SDHCI_TRNS_READ;
+#ifdef MMCCAM
+	struct ccb_mmcio *mmcio;
+	mmcio = &slot->ccb->mmcio;
+	if (mmcio->stop.opcode == MMC_STOP_TRANSMISSION
+	    && !(slot->quirks & SDHCI_QUIRK_BROKEN_AUTO_STOP))
+		mode |= SDHCI_TRNS_ACMD12;
+#else
 	if (slot->req->stop && !(slot->quirks & SDHCI_QUIRK_BROKEN_AUTO_STOP))
 		mode |= SDHCI_TRNS_ACMD12;
+#endif
 	if (slot->flags & SDHCI_USE_DMA)
 		mode |= SDHCI_TRNS_DMA;
 
@@ -1096,6 +1592,9 @@ sdhci_start_command(struct sdhci_slot *slot, struct mmc_command *cmd)
 	if (!SDHCI_GET_CARD_PRESENT(slot->bus, slot) ||
 	    slot->power == 0 ||
 	    slot->clock == 0) {
+		slot_printf(slot,
+			    "Cannot issue a command (power=%d clock=%d)",
+			    slot->power, slot->clock);
 		cmd->error = MMC_ERR_FAILED;
 		sdhci_req_done(slot);
 		return;
@@ -1103,10 +1602,20 @@ sdhci_start_command(struct sdhci_slot *slot, struct mmc_command *cmd)
 	/* Always wait for free CMD bus. */
 	mask = SDHCI_CMD_INHIBIT;
 	/* Wait for free DAT if we have data or busy signal. */
-	if (cmd->data || (cmd->flags & MMC_RSP_BUSY))
+	if (cmd->data != NULL || (cmd->flags & MMC_RSP_BUSY))
 		mask |= SDHCI_DAT_INHIBIT;
-	/* We shouldn't wait for DAT for stop commands. */
-	if (cmd == slot->req->stop)
+	/*
+	 * We shouldn't wait for DAT for stop commands or CMD19/CMD21.  Note
+	 * that these latter are also special in that SDHCI_CMD_DATA should
+	 * be set below but no actual data is ever read from the controller.
+	*/
+#ifdef MMCCAM
+	if (cmd == &slot->ccb->mmcio.stop ||
+#else
+	if (cmd == slot->req->stop ||
+#endif
+	    __predict_false(cmd->opcode == MMC_SEND_TUNING_BLOCK ||
+	    cmd->opcode == MMC_SEND_TUNING_BLOCK_HS200))
 		mask &= ~SDHCI_DAT_INHIBIT;
 	/*
 	 *  Wait for bus no more then 250 ms.  Typically there will be no wait
@@ -1145,7 +1654,7 @@ sdhci_start_command(struct sdhci_slot *slot, struct mmc_command *cmd)
 		flags |= SDHCI_CMD_CRC;
 	if (cmd->flags & MMC_RSP_OPCODE)
 		flags |= SDHCI_CMD_INDEX;
-	if (cmd->data)
+	if (cmd->data != NULL)
 		flags |= SDHCI_CMD_DATA;
 	if (cmd->opcode == MMC_STOP_TRANSMISSION)
 		flags |= SDHCI_CMD_TYPE_ABORT;
@@ -1164,6 +1673,8 @@ sdhci_start_command(struct sdhci_slot *slot, struct mmc_command *cmd)
 	WR4(slot, SDHCI_ARGUMENT, cmd->arg);
 	/* Set data transfer mode. */
 	sdhci_set_transfer_mode(slot, cmd->data);
+	if (__predict_false(sdhci_debug > 1))
+		slot_printf(slot, "Starting command!\n");
 	/* Start command. */
 	WR2(slot, SDHCI_COMMAND_FLAGS, (cmd->opcode << 8) | (flags & 0xff));
 	/* Start timeout callout. */
@@ -1178,15 +1689,23 @@ sdhci_finish_command(struct sdhci_slot *slot)
 	uint32_t val;
 	uint8_t extra;
 
+	if (__predict_false(sdhci_debug > 1))
+		slot_printf(slot, "%s: called, err %d flags %d\n",
+		    __func__, slot->curcmd->error, slot->curcmd->flags);
 	slot->cmd_done = 1;
 	/*
 	 * Interrupt aggregation: Restore command interrupt.
 	 * Main restore point for the case when command interrupt
 	 * happened first.
 	 */
-	WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask |= SDHCI_INT_RESPONSE);
+	if (__predict_true(slot->curcmd->opcode != MMC_SEND_TUNING_BLOCK &&
+	    slot->curcmd->opcode != MMC_SEND_TUNING_BLOCK_HS200))
+		WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask |=
+		    SDHCI_INT_RESPONSE);
 	/* In case of error - reset host and return. */
 	if (slot->curcmd->error) {
+		if (slot->curcmd->error == MMC_ERR_BADCRC)
+			slot->retune_req |= SDHCI_RETUNE_REQ_RESET;
 		sdhci_reset(slot, SDHCI_RESET_CMD);
 		sdhci_reset(slot, SDHCI_RESET_DATA);
 		sdhci_start(slot);
@@ -1211,6 +1730,11 @@ sdhci_finish_command(struct sdhci_slot *slot)
 		} else
 			slot->curcmd->resp[0] = RD4(slot, SDHCI_RESPONSE);
 	}
+	if (__predict_false(sdhci_debug > 1))
+		printf("Resp: %02x %02x %02x %02x\n",
+		    slot->curcmd->resp[0], slot->curcmd->resp[1],
+		    slot->curcmd->resp[2], slot->curcmd->resp[3]);
+
 	/* If data ready - finish. */
 	if (slot->data_done)
 		sdhci_start(slot);
@@ -1291,6 +1815,11 @@ sdhci_start_data(struct sdhci_slot *slot, struct mmc_data *data)
 	    (data->len < 512) ? data->len : 512));
 	/* Set block count. */
 	WR2(slot, SDHCI_BLOCK_COUNT, (data->len + 511) / 512);
+
+	if (__predict_false(sdhci_debug > 1))
+		slot_printf(slot, "Block size: %02x, count %lu\n",
+		    (unsigned int)SDHCI_MAKE_BLKSZ(DMA_BOUNDARY, (data->len < 512) ? data->len : 512),
+		    (unsigned long)(data->len + 511) / 512);
 }
 
 void
@@ -1322,6 +1851,8 @@ sdhci_finish_data(struct sdhci_slot *slot)
 	slot->data_done = 1;
 	/* If there was error - reset the host. */
 	if (slot->curcmd->error) {
+		if (slot->curcmd->error == MMC_ERR_BADCRC)
+			slot->retune_req |= SDHCI_RETUNE_REQ_RESET;
 		sdhci_reset(slot, SDHCI_RESET_CMD);
 		sdhci_reset(slot, SDHCI_RESET_DATA);
 		sdhci_start(slot);
@@ -1332,6 +1863,47 @@ sdhci_finish_data(struct sdhci_slot *slot)
 		sdhci_start(slot);
 }
 
+#ifdef MMCCAM
+static void
+sdhci_start(struct sdhci_slot *slot)
+{
+        union ccb *ccb;
+
+	ccb = slot->ccb;
+	if (ccb == NULL)
+		return;
+
+        struct ccb_mmcio *mmcio;
+	mmcio = &ccb->mmcio;
+
+	if (!(slot->flags & CMD_STARTED)) {
+		slot->flags |= CMD_STARTED;
+		sdhci_start_command(slot, &mmcio->cmd);
+		return;
+	}
+
+	/*
+	 * Old stack doesn't use this!
+	 * Enabling this code causes significant performance degradation
+	 * and IRQ storms on BBB, Wandboard behaves fine.
+	 * Not using this code does no harm...
+	if (!(slot->flags & STOP_STARTED) && mmcio->stop.opcode != 0) {
+		slot->flags |= STOP_STARTED;
+		sdhci_start_command(slot, &mmcio->stop);
+		return;
+	}
+	*/
+	if (__predict_false(sdhci_debug > 1))
+		slot_printf(slot, "result: %d\n", mmcio->cmd.error);
+	if (mmcio->cmd.error == 0 &&
+	    (slot->quirks & SDHCI_QUIRK_RESET_AFTER_REQUEST)) {
+		sdhci_reset(slot, SDHCI_RESET_CMD);
+		sdhci_reset(slot, SDHCI_RESET_DATA);
+	}
+
+	sdhci_req_done(slot);
+}
+#else
 static void
 sdhci_start(struct sdhci_slot *slot)
 {
@@ -1352,7 +1924,7 @@ sdhci_start(struct sdhci_slot *slot)
 		sdhci_start_command(slot, req->stop);
 		return;
 	}
-	if (sdhci_debug > 1)
+	if (__predict_false(sdhci_debug > 1))
 		slot_printf(slot, "result: %d\n", req->cmd->error);
 	if (!req->cmd->error &&
 	    ((slot->curcmd == req->stop &&
@@ -1364,6 +1936,7 @@ sdhci_start(struct sdhci_slot *slot)
 
 	sdhci_req_done(slot);
 }
+#endif
 
 int
 sdhci_generic_request(device_t brdev __unused, device_t reqdev,
@@ -1376,7 +1949,7 @@ sdhci_generic_request(device_t brdev __unused, device_t reqdev,
 		SDHCI_UNLOCK(slot);
 		return (EBUSY);
 	}
-	if (sdhci_debug > 1) {
+	if (__predict_false(sdhci_debug > 1)) {
 		slot_printf(slot,
 		    "CMD%u arg %#x flags %#x dlen %u dflags %#x\n",
 		    req->cmd->opcode, req->cmd->arg, req->cmd->flags,
@@ -1495,6 +2068,15 @@ sdhci_data_irq(struct sdhci_slot *slot, uint32_t intmask)
 		goto done;
 	}
 
+	/* Handle tuning completion interrupt. */
+	if (__predict_false((intmask & SDHCI_INT_DATA_AVAIL) &&
+	    (slot->curcmd->opcode == MMC_SEND_TUNING_BLOCK ||
+	    slot->curcmd->opcode == MMC_SEND_TUNING_BLOCK_HS200))) {
+		slot->req->flags |= MMC_TUNE_DONE;
+		sdhci_finish_command(slot);
+		sdhci_finish_data(slot);
+		return;
+	}
 	/* Handle PIO interrupt. */
 	if (intmask & (SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL)) {
 		if ((slot->opt & SDHCI_PLATFORM_TRANSFER) &&
@@ -1587,9 +2169,21 @@ sdhci_generic_intr(struct sdhci_slot *slot)
 		SDHCI_UNLOCK(slot);
 		return;
 	}
-	if (sdhci_debug > 2)
+	if (__predict_false(sdhci_debug > 2))
 		slot_printf(slot, "Interrupt %#x\n", intmask);
 
+	/* Handle tuning error interrupt. */
+	if (__predict_false(intmask & SDHCI_INT_TUNEERR)) {
+		slot_printf(slot, "Tuning error indicated\n");
+		slot->retune_req |= SDHCI_RETUNE_REQ_RESET;
+		if (slot->curcmd) {
+			slot->curcmd->error = MMC_ERR_BADCRC;
+			sdhci_finish_command(slot);
+		}
+	}
+	/* Handle re-tuning interrupt. */
+	if (__predict_false(intmask & SDHCI_INT_RETUNE))
+		slot->retune_req |= SDHCI_RETUNE_REQ_NEEDED;
 	/* Handle card presence interrupts. */
 	if (intmask & (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE)) {
 		present = (intmask & SDHCI_INT_CARD_INSERT) != 0;
@@ -1602,7 +2196,6 @@ sdhci_generic_intr(struct sdhci_slot *slot)
 		WR4(slot, SDHCI_INT_STATUS, intmask &
 		    (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE));
 		sdhci_handle_card_present_locked(slot, present);
-		intmask &= ~(SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE);
 	}
 	/* Handle command interrupts. */
 	if (intmask & SDHCI_INT_CMD_MASK) {
@@ -1621,16 +2214,14 @@ sdhci_generic_intr(struct sdhci_slot *slot)
 		WR4(slot, SDHCI_INT_STATUS, SDHCI_INT_ACMD12ERR);
 		sdhci_acmd_irq(slot);
 	}
-	intmask &= ~(SDHCI_INT_CMD_MASK | SDHCI_INT_DATA_MASK);
-	intmask &= ~SDHCI_INT_ACMD12ERR;
-	intmask &= ~SDHCI_INT_ERROR;
 	/* Handle bus power interrupt. */
 	if (intmask & SDHCI_INT_BUS_POWER) {
 		WR4(slot, SDHCI_INT_STATUS, SDHCI_INT_BUS_POWER);
-		slot_printf(slot,
-		    "Card is consuming too much power!\n");
-		intmask &= ~SDHCI_INT_BUS_POWER;
+		slot_printf(slot, "Card is consuming too much power!\n");
 	}
+	intmask &= ~(SDHCI_INT_ERROR | SDHCI_INT_TUNEERR | SDHCI_INT_RETUNE |
+	    SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE | SDHCI_INT_CMD_MASK |
+	    SDHCI_INT_DATA_MASK | SDHCI_INT_ACMD12ERR | SDHCI_INT_BUS_POWER);
 	/* The rest is unknown. */
 	if (intmask) {
 		WR4(slot, SDHCI_INT_STATUS, intmask);
@@ -1684,6 +2275,19 @@ sdhci_generic_read_ivar(device_t bus, device_t child, int which,
 	case MMCBR_IVAR_VDD:
 		*result = slot->host.ios.vdd;
 		break;
+	case MMCBR_IVAR_RETUNE_REQ:
+		if (slot->opt & SDHCI_TUNING_ENABLED) {
+			if (slot->retune_req & SDHCI_RETUNE_REQ_RESET) {
+				*result = retune_req_reset;
+				break;
+			}
+			if (slot->retune_req & SDHCI_RETUNE_REQ_NEEDED) {
+				*result = retune_req_normal;
+				break;
+			}
+		}
+		*result = retune_req_none;
+		break;
 	case MMCBR_IVAR_VCCQ:
 		*result = slot->host.ios.vccq;
 		break;
@@ -1694,6 +2298,16 @@ sdhci_generic_read_ivar(device_t bus, device_t child, int which,
 		*result = slot->host.ios.timing;
 		break;
 	case MMCBR_IVAR_MAX_DATA:
+		/*
+		 * Re-tuning modes 1 and 2 restrict the maximum data length
+		 * per read/write command to 4 MiB.
+		 */
+		if (slot->opt & SDHCI_TUNING_ENABLED &&
+		    (slot->retune_mode == SDHCI_RETUNE_MODE_1 ||
+		    slot->retune_mode == SDHCI_RETUNE_MODE_2)) {
+			*result = 4 * 1024 * 1024 / MMC_SECTOR_SIZE;
+			break;
+		}
 		*result = 65535;
 		break;
 	case MMCBR_IVAR_MAX_BUSY_TIMEOUT:
@@ -1714,6 +2328,8 @@ sdhci_generic_write_ivar(device_t bus, device_t child, int which,
 	uint32_t clock, max_clock;
 	int i;
 
+	if (sdhci_debug > 1)
+		slot_printf(slot, "%s: var=%d\n", __func__, which);
 	switch (which) {
 	default:
 		return (EINVAL);
@@ -1774,9 +2390,324 @@ sdhci_generic_write_ivar(device_t bus, device_t child, int which,
 	case MMCBR_IVAR_F_MIN:
 	case MMCBR_IVAR_F_MAX:
 	case MMCBR_IVAR_MAX_DATA:
+	case MMCBR_IVAR_RETUNE_REQ:
 		return (EINVAL);
 	}
 	return (0);
 }
 
+#ifdef MMCCAM
+void
+sdhci_cam_start_slot(struct sdhci_slot *slot)
+{
+        if ((slot->devq = cam_simq_alloc(1)) == NULL) {
+                goto fail;
+        }
+
+        mtx_init(&slot->sim_mtx, "sdhcisim", NULL, MTX_DEF);
+        slot->sim = cam_sim_alloc(sdhci_cam_action, sdhci_cam_poll,
+                                  "sdhci_slot", slot, device_get_unit(slot->bus),
+                                  &slot->sim_mtx, 1, 1, slot->devq);
+
+        if (slot->sim == NULL) {
+                cam_simq_free(slot->devq);
+                slot_printf(slot, "cannot allocate CAM SIM\n");
+                goto fail;
+        }
+
+        mtx_lock(&slot->sim_mtx);
+        if (xpt_bus_register(slot->sim, slot->bus, 0) != 0) {
+                slot_printf(slot,
+                              "cannot register SCSI pass-through bus\n");
+                cam_sim_free(slot->sim, FALSE);
+                cam_simq_free(slot->devq);
+                mtx_unlock(&slot->sim_mtx);
+                goto fail;
+        }
+
+        mtx_unlock(&slot->sim_mtx);
+        /* End CAM-specific init */
+	slot->card_present = 0;
+	sdhci_card_task(slot, 0);
+        return;
+
+fail:
+        if (slot->sim != NULL) {
+                mtx_lock(&slot->sim_mtx);
+                xpt_bus_deregister(cam_sim_path(slot->sim));
+                cam_sim_free(slot->sim, FALSE);
+                mtx_unlock(&slot->sim_mtx);
+        }
+
+        if (slot->devq != NULL)
+                cam_simq_free(slot->devq);
+}
+
+static void
+sdhci_cam_handle_mmcio(struct cam_sim *sim, union ccb *ccb)
+{
+	struct sdhci_slot *slot;
+
+	slot = cam_sim_softc(sim);
+
+	sdhci_cam_request(slot, ccb);
+}
+
+void
+sdhci_cam_action(struct cam_sim *sim, union ccb *ccb)
+{
+	struct sdhci_slot *slot;
+
+	slot = cam_sim_softc(sim);
+	if (slot == NULL) {
+		ccb->ccb_h.status = CAM_SEL_TIMEOUT;
+		xpt_done(ccb);
+		return;
+	}
+
+	mtx_assert(&slot->sim_mtx, MA_OWNED);
+
+	switch (ccb->ccb_h.func_code) {
+	case XPT_PATH_INQ:
+	{
+		struct ccb_pathinq *cpi;
+
+		cpi = &ccb->cpi;
+		cpi->version_num = 1;
+		cpi->hba_inquiry = 0;
+		cpi->target_sprt = 0;
+		cpi->hba_misc = PIM_NOBUSRESET | PIM_SEQSCAN;
+		cpi->hba_eng_cnt = 0;
+		cpi->max_target = 0;
+		cpi->max_lun = 0;
+		cpi->initiator_id = 1;
+		cpi->maxio = MAXPHYS;
+		strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
+		strncpy(cpi->hba_vid, "Deglitch Networks", HBA_IDLEN);
+		strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
+		cpi->unit_number = cam_sim_unit(sim);
+		cpi->bus_id = cam_sim_bus(sim);
+		cpi->base_transfer_speed = 100; /* XXX WTF? */
+		cpi->protocol = PROTO_MMCSD;
+		cpi->protocol_version = SCSI_REV_0;
+		cpi->transport = XPORT_MMCSD;
+		cpi->transport_version = 0;
+
+		cpi->ccb_h.status = CAM_REQ_CMP;
+		break;
+	}
+	case XPT_GET_TRAN_SETTINGS:
+	{
+		struct ccb_trans_settings *cts = &ccb->cts;
+
+		if (sdhci_debug > 1)
+			slot_printf(slot, "Got XPT_GET_TRAN_SETTINGS\n");
+
+		cts->protocol = PROTO_MMCSD;
+		cts->protocol_version = 1;
+		cts->transport = XPORT_MMCSD;
+		cts->transport_version = 1;
+		cts->xport_specific.valid = 0;
+		cts->proto_specific.mmc.host_ocr = slot->host.host_ocr;
+		cts->proto_specific.mmc.host_f_min = slot->host.f_min;
+		cts->proto_specific.mmc.host_f_max = slot->host.f_max;
+		cts->proto_specific.mmc.host_caps = slot->host.caps;
+		memcpy(&cts->proto_specific.mmc.ios, &slot->host.ios, sizeof(struct mmc_ios));
+		ccb->ccb_h.status = CAM_REQ_CMP;
+		break;
+	}
+	case XPT_SET_TRAN_SETTINGS:
+	{
+		if (sdhci_debug > 1)
+			slot_printf(slot, "Got XPT_SET_TRAN_SETTINGS\n");
+		sdhci_cam_settran_settings(slot, ccb);
+		ccb->ccb_h.status = CAM_REQ_CMP;
+		break;
+	}
+	case XPT_RESET_BUS:
+		if (sdhci_debug > 1)
+			slot_printf(slot, "Got XPT_RESET_BUS, ACK it...\n");
+		ccb->ccb_h.status = CAM_REQ_CMP;
+		break;
+	case XPT_MMC_IO:
+		/*
+		 * Here is the HW-dependent part of
+		 * sending the command to the underlying h/w
+		 * At some point in the future an interrupt comes.
+		 * Then the request will be marked as completed.
+		 */
+		if (__predict_false(sdhci_debug > 1))
+			slot_printf(slot, "Got XPT_MMC_IO\n");
+		ccb->ccb_h.status = CAM_REQ_INPROG;
+
+		sdhci_cam_handle_mmcio(sim, ccb);
+		return;
+		/* NOTREACHED */
+		break;
+	default:
+		ccb->ccb_h.status = CAM_REQ_INVALID;
+		break;
+	}
+	xpt_done(ccb);
+	return;
+}
+
+void
+sdhci_cam_poll(struct cam_sim *sim)
+{
+	return;
+}
+
+int sdhci_cam_get_possible_host_clock(struct sdhci_slot *slot, int proposed_clock) {
+	int max_clock, clock, i;
+
+	if (proposed_clock == 0)
+		return 0;
+	max_clock = slot->max_clk;
+	clock = max_clock;
+
+	if (slot->version < SDHCI_SPEC_300) {
+		for (i = 0; i < SDHCI_200_MAX_DIVIDER;
+		     i <<= 1) {
+			if (clock <= proposed_clock)
+				break;
+			clock >>= 1;
+		}
+	} else {
+		for (i = 0; i < SDHCI_300_MAX_DIVIDER;
+		     i += 2) {
+			if (clock <= proposed_clock)
+				break;
+			clock = max_clock / (i + 2);
+		}
+	}
+	return clock;
+}
+
+int
+sdhci_cam_settran_settings(struct sdhci_slot *slot, union ccb *ccb)
+{
+	struct mmc_ios *ios;
+	struct mmc_ios *new_ios;
+	struct ccb_trans_settings_mmc *cts;
+
+	ios = &slot->host.ios;
+
+	cts = &ccb->cts.proto_specific.mmc;
+	new_ios = &cts->ios;
+
+	/* Update only requested fields */
+	if (cts->ios_valid & MMC_CLK) {
+		ios->clock = sdhci_cam_get_possible_host_clock(slot, new_ios->clock);
+		slot_printf(slot, "Clock => %d\n", ios->clock);
+	}
+	if (cts->ios_valid & MMC_VDD) {
+		ios->vdd = new_ios->vdd;
+		slot_printf(slot, "VDD => %d\n", ios->vdd);
+	}
+	if (cts->ios_valid & MMC_CS) {
+		ios->chip_select = new_ios->chip_select;
+		slot_printf(slot, "CS => %d\n", ios->chip_select);
+	}
+	if (cts->ios_valid & MMC_BW) {
+		ios->bus_width = new_ios->bus_width;
+		slot_printf(slot, "Bus width => %d\n", ios->bus_width);
+	}
+	if (cts->ios_valid & MMC_PM) {
+		ios->power_mode = new_ios->power_mode;
+		slot_printf(slot, "Power mode => %d\n", ios->power_mode);
+	}
+	if (cts->ios_valid & MMC_BT) {
+		ios->timing = new_ios->timing;
+		slot_printf(slot, "Timing => %d\n", ios->timing);
+	}
+	if (cts->ios_valid & MMC_BM) {
+		ios->bus_mode = new_ios->bus_mode;
+		slot_printf(slot, "Bus mode => %d\n", ios->bus_mode);
+	}
+
+        /* XXX Provide a way to call a chip-specific IOS update, required for TI */
+	return (sdhci_cam_update_ios(slot));
+}
+
+int
+sdhci_cam_update_ios(struct sdhci_slot *slot)
+{
+	struct mmc_ios *ios = &slot->host.ios;
+
+	slot_printf(slot, "%s: power_mode=%d, clk=%d, bus_width=%d, timing=%d\n",
+		    __func__, ios->power_mode, ios->clock, ios->bus_width, ios->timing);
+	SDHCI_LOCK(slot);
+	/* Do full reset on bus power down to clear from any state. */
+	if (ios->power_mode == power_off) {
+		WR4(slot, SDHCI_SIGNAL_ENABLE, 0);
+		sdhci_init(slot);
+	}
+	/* Configure the bus. */
+	sdhci_set_clock(slot, ios->clock);
+	sdhci_set_power(slot, (ios->power_mode == power_off) ? 0 : ios->vdd);
+	if (ios->bus_width == bus_width_8) {
+		slot->hostctrl |= SDHCI_CTRL_8BITBUS;
+		slot->hostctrl &= ~SDHCI_CTRL_4BITBUS;
+	} else if (ios->bus_width == bus_width_4) {
+		slot->hostctrl &= ~SDHCI_CTRL_8BITBUS;
+		slot->hostctrl |= SDHCI_CTRL_4BITBUS;
+	} else if (ios->bus_width == bus_width_1) {
+		slot->hostctrl &= ~SDHCI_CTRL_8BITBUS;
+		slot->hostctrl &= ~SDHCI_CTRL_4BITBUS;
+	} else {
+		panic("Invalid bus width: %d", ios->bus_width);
+	}
+	if (ios->timing == bus_timing_hs &&
+	    !(slot->quirks & SDHCI_QUIRK_DONT_SET_HISPD_BIT))
+		slot->hostctrl |= SDHCI_CTRL_HISPD;
+	else
+		slot->hostctrl &= ~SDHCI_CTRL_HISPD;
+	WR1(slot, SDHCI_HOST_CONTROL, slot->hostctrl);
+	/* Some controllers like reset after bus changes. */
+	if(slot->quirks & SDHCI_QUIRK_RESET_ON_IOS)
+		sdhci_reset(slot, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
+
+	SDHCI_UNLOCK(slot);
+	return (0);
+}
+
+int
+sdhci_cam_request(struct sdhci_slot *slot, union ccb *ccb)
+{
+	struct ccb_mmcio *mmcio;
+
+	mmcio = &ccb->mmcio;
+
+	SDHCI_LOCK(slot);
+/*	if (slot->req != NULL) {
+		SDHCI_UNLOCK(slot);
+		return (EBUSY);
+	}
+*/
+	if (__predict_false(sdhci_debug > 1)) {
+		slot_printf(slot, "CMD%u arg %#x flags %#x dlen %u dflags %#x\n",
+			    mmcio->cmd.opcode, mmcio->cmd.arg, mmcio->cmd.flags,
+			    mmcio->cmd.data != NULL ? (unsigned int) mmcio->cmd.data->len : 0,
+			    mmcio->cmd.data != NULL ? mmcio->cmd.data->flags: 0);
+	}
+	if (mmcio->cmd.data != NULL) {
+		if (mmcio->cmd.data->len == 0 || mmcio->cmd.data->flags == 0)
+			panic("data->len = %d, data->flags = %d -- something is b0rked",
+			      (int)mmcio->cmd.data->len, mmcio->cmd.data->flags);
+	}
+	slot->ccb = ccb;
+	slot->flags = 0;
+	sdhci_start(slot);
+	SDHCI_UNLOCK(slot);
+	if (dumping) {
+		while (slot->ccb != NULL) {
+			sdhci_generic_intr(slot);
+			DELAY(10);
+		}
+	}
+	return (0);
+}
+#endif /* MMCCAM */
+
 MODULE_VERSION(sdhci, 1);
diff --git a/freebsd/sys/dev/sdhci/sdhci.h b/freebsd/sys/dev/sdhci/sdhci.h
index 814f81ed..73aa84b6 100644
--- a/freebsd/sys/dev/sdhci/sdhci.h
+++ b/freebsd/sys/dev/sdhci/sdhci.h
@@ -28,6 +28,8 @@
 #ifndef	__SDHCI_H__
 #define	__SDHCI_H__
 
+#include <rtems/bsd/local/opt_mmccam.h>
+
 #define	DMA_BLOCK_SIZE	4096
 #define	DMA_BOUNDARY	0	/* DMA reload every 4K */
 
@@ -235,6 +237,11 @@
 #define	SDHCI_HOST_CONTROL2	0x3E
 #define	 SDHCI_CTRL2_PRESET_VALUE	0x8000
 #define	 SDHCI_CTRL2_ASYNC_INTR	0x4000
+#define	 SDHCI_CTRL2_64BIT_ENABLE	0x2000
+#define	 SDHCI_CTRL2_HOST_V4_ENABLE	0x1000
+#define	 SDHCI_CTRL2_CMD23_ENABLE	0x0800
+#define	 SDHCI_CTRL2_ADMA2_LENGTH_MODE	0x0400
+#define	 SDHCI_CTRL2_UHS2_IFACE_ENABLE	0x0100
 #define	 SDHCI_CTRL2_SAMPLING_CLOCK	0x0080
 #define	 SDHCI_CTRL2_EXEC_TUNING	0x0040
 #define	 SDHCI_CTRL2_DRIVER_TYPE_MASK	0x0030
@@ -319,6 +326,8 @@
 #define	SDHCI_SPEC_200		1
 #define	SDHCI_SPEC_300		2
 #define	SDHCI_SPEC_400		3
+#define	SDHCI_SPEC_410		4
+#define	SDHCI_SPEC_420		5
 
 SYSCTL_DECL(_hw_sdhci);
 
@@ -326,6 +335,7 @@ extern u_int sdhci_quirk_clear;
 extern u_int sdhci_quirk_set;
 
 struct sdhci_slot {
+	struct mtx	mtx;		/* Slot mutex */
 	u_int		quirks;		/* Chip specific quirks */
 	u_int		caps;		/* Override SDHCI_CAPABILITIES */
 	u_int		caps2;		/* Override SDHCI_CAPABILITIES2 */
@@ -336,6 +346,10 @@ struct sdhci_slot {
 #define	SDHCI_HAVE_DMA			0x01
 #define	SDHCI_PLATFORM_TRANSFER		0x02
 #define	SDHCI_NON_REMOVABLE		0x04
+#define	SDHCI_TUNING_SUPPORTED		0x08
+#define	SDHCI_TUNING_ENABLED		0x10
+#define	SDHCI_SDR50_NEEDS_TUNING	0x20
+#define	SDHCI_SLOT_EMBEDDED		0x40
 	u_char		version;
 	int		timeout;	/* Transfer timeout */
 	uint32_t	max_clk;	/* Max possible freq */
@@ -349,14 +363,27 @@ struct sdhci_slot {
 			card_delayed_task;/* Card insert delayed task */
 	struct callout	card_poll_callout;/* Card present polling callout */
 	struct callout	timeout_callout;/* Card command/data response timeout */
+	struct callout	retune_callout;	/* Re-tuning mode 1 callout */
 	struct mmc_host host;		/* Host parameters */
 	struct mmc_request *req;	/* Current request */
 	struct mmc_command *curcmd;	/* Current command of current request */
 
+	struct mmc_request *tune_req;	/* Tuning request */
+	struct mmc_command *tune_cmd;	/* Tuning command of tuning request */
+	struct mmc_data *tune_data;	/* Tuning data of tuning command */
+	uint32_t	retune_ticks;	/* Re-tuning callout ticks [hz] */
 	uint32_t	intmask;	/* Current interrupt mask */
 	uint32_t	clock;		/* Current clock freq. */
 	size_t		offset;		/* Data buffer offset */
 	uint8_t		hostctrl;	/* Current host control register */
+	uint8_t		retune_count;	/* Controller re-tuning count [s] */
+	uint8_t		retune_mode;	/* Controller re-tuning mode */
+#define	SDHCI_RETUNE_MODE_1	0x00
+#define	SDHCI_RETUNE_MODE_2	0x01
+#define	SDHCI_RETUNE_MODE_3	0x02
+	uint8_t		retune_req;	/* Re-tuning request status */
+#define	SDHCI_RETUNE_REQ_NEEDED	0x01	/* Re-tuning w/o circuit reset needed */
+#define	SDHCI_RETUNE_REQ_RESET	0x02	/* Re-tuning w/ circuit reset needed */
 	u_char		power;		/* Current power */
 	u_char		bus_busy;	/* Bus busy status */
 	u_char		cmd_done;	/* CMD command part done flag */
@@ -366,7 +393,15 @@ struct sdhci_slot {
 #define	STOP_STARTED		2
 #define	SDHCI_USE_DMA		4	/* Use DMA for this req. */
 #define	PLATFORM_DATA_STARTED	8	/* Data xfer is handled by platform */
-	struct mtx	mtx;		/* Slot mutex */
+
+#ifdef MMCCAM
+	/* CAM stuff */
+	union ccb	*ccb;
+	struct cam_devq		*devq;
+	struct cam_sim		*sim;
+	struct mtx		sim_mtx;
+	u_char			card_present; /* XXX Maybe derive this from elsewhere? */
+#endif
 };
 
 int sdhci_generic_read_ivar(device_t bus, device_t child, int which,
@@ -381,7 +416,9 @@ int sdhci_cleanup_slot(struct sdhci_slot *slot);
 int sdhci_generic_suspend(struct sdhci_slot *slot);
 int sdhci_generic_resume(struct sdhci_slot *slot);
 int sdhci_generic_update_ios(device_t brdev, device_t reqdev);
+int sdhci_generic_tune(device_t brdev, device_t reqdev, bool hs400);
 int sdhci_generic_switch_vccq(device_t brdev, device_t reqdev);
+int sdhci_generic_retune(device_t brdev, device_t reqdev, bool reset);
 int sdhci_generic_request(device_t brdev, device_t reqdev,
     struct mmc_request *req);
 int sdhci_generic_get_ro(device_t brdev, device_t reqdev);
@@ -393,4 +430,9 @@ bool sdhci_generic_get_card_present(device_t brdev, struct sdhci_slot *slot);
 void sdhci_generic_set_uhs_timing(device_t brdev, struct sdhci_slot *slot);
 void sdhci_handle_card_present(struct sdhci_slot *slot, bool is_present);
 
+#ifdef MMCCAM
+/* CAM-related */
+void sdhci_cam_start_slot(struct sdhci_slot *slot);
+#endif
+
 #endif	/* __SDHCI_H__ */
diff --git a/freebsd/sys/dev/tsec/if_tsec.c b/freebsd/sys/dev/tsec/if_tsec.c
index d76c8f10..b7edc948 100644
--- a/freebsd/sys/dev/tsec/if_tsec.c
+++ b/freebsd/sys/dev/tsec/if_tsec.c
@@ -39,7 +39,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bus.h>
-#include <sys/bus_dma.h>
 #include <sys/endian.h>
 #include <sys/mbuf.h>
 #include <sys/kernel.h>
diff --git a/freebsd/sys/dev/usb/controller/saf1761_otg.c b/freebsd/sys/dev/usb/controller/saf1761_otg.c
index f9d28a63..e1ce5eb6 100644
--- a/freebsd/sys/dev/usb/controller/saf1761_otg.c
+++ b/freebsd/sys/dev/usb/controller/saf1761_otg.c
@@ -525,9 +525,6 @@ saf1761_host_bulk_data_rx(struct saf1761_otg_softc *sc, struct saf1761_otg_td *t
 		DPRINTFN(5, "STATUS=0x%08x\n", status);
 
 		if (status & SOTG_PTD_DW3_ACTIVE) {
-#ifndef __rtems__
-			goto busy;
-#else /* __rtems__ */
 			temp = saf1761_peek_host_status_le_4(sc,
 			    pdt_addr + SOTG_PTD_DW0);
 			if (temp & SOTG_PTD_DW0_VALID) {
@@ -547,7 +544,6 @@ saf1761_host_bulk_data_rx(struct saf1761_otg_softc *sc, struct saf1761_otg_td *t
 					goto complete;
 				}
 			}
-#endif /* __rtems__ */
 		} else if (status & SOTG_PTD_DW3_HALTED) {
 			if (!(status & SOTG_PTD_DW3_ERRORS))
 				td->error_stall = 1;
@@ -591,9 +587,7 @@ saf1761_host_bulk_data_rx(struct saf1761_otg_softc *sc, struct saf1761_otg_td *t
 		}
 		saf1761_host_channel_free(sc, td);
 	}
-#ifdef __rtems__
 retry:
-#endif /* __rtems__ */
 	if (saf1761_host_channel_alloc(sc, td))
 		goto busy;
 
@@ -1623,10 +1617,8 @@ saf1761_otg_filter_interrupt(void *arg)
 	(void) SAF1761_READ_LE_4(sc, SOTG_INT_PTD_DONE_PTD);
 	(void) SAF1761_READ_LE_4(sc, SOTG_ISO_PTD_DONE_PTD);
 
-#ifdef __rtems__
 	DPRINTFN(9, "HCINTERRUPT=0x%08x DCINTERRUPT=0x%08x\n", hcstat, status);
 
-#endif /* __rtems__ */
 	if (status & SOTG_DCINTERRUPT_IEPSOF) {
 		if ((sc->sc_host_async_busy_map[1] | sc->sc_host_async_busy_map[0] |
 		     sc->sc_host_intr_busy_map[1] | sc->sc_host_intr_busy_map[0] |
@@ -2496,13 +2488,12 @@ saf1761_otg_init(struct saf1761_otg_softc *sc)
 #ifdef __rtems__
 	SAF1761_WRITE_LE_4(sc, SOTG_CTRL_SET_CLR,
 	    SOTG_CTRL_SET(SOTG_CTRL_SEL_CP_EXT | SOTG_CTRL_VBUS_DRV));
-#else /* __rtems__ */
+#else
 	SAF1761_WRITE_LE_4(sc, SOTG_CTRL_SET_CLR,
 	    SOTG_CTRL_SET(SOTG_CTRL_SW_SEL_HC_DC |
 	    SOTG_CTRL_BDIS_ACON_EN | SOTG_CTRL_SEL_CP_EXT |
 	    SOTG_CTRL_VBUS_DRV));
-#endif /* __rtems__ */
-
+#endif
 	/* disable device address */
 	SAF1761_WRITE_LE_4(sc, SOTG_ADDRESS, 0);
 
diff --git a/freebsd/sys/dev/usb/wlan/if_rsu.c b/freebsd/sys/dev/usb/wlan/if_rsu.c
index 659e178a..2415d2f2 100644
--- a/freebsd/sys/dev/usb/wlan/if_rsu.c
+++ b/freebsd/sys/dev/usb/wlan/if_rsu.c
@@ -2036,6 +2036,8 @@ rsu_hwrssi_to_rssi(struct rsu_softc *sc, int hw_rssi)
 	return (v);
 }
 
+CTASSERT(MCLBYTES > sizeof(struct ieee80211_frame));
+
 static void
 rsu_event_survey(struct rsu_softc *sc, uint8_t *buf, int len)
 {
@@ -2044,28 +2046,31 @@ rsu_event_survey(struct rsu_softc *sc, uint8_t *buf, int len)
 	struct ndis_wlan_bssid_ex *bss;
 	struct ieee80211_rx_stats rxs;
 	struct mbuf *m;
-	int pktlen;
+	uint32_t ieslen;
+	uint32_t pktlen;
 
 	if (__predict_false(len < sizeof(*bss)))
 		return;
 	bss = (struct ndis_wlan_bssid_ex *)buf;
-	if (__predict_false(len < sizeof(*bss) + le32toh(bss->ieslen)))
+	ieslen = le32toh(bss->ieslen);
+	/* range check length of information element */
+	if (__predict_false(ieslen > (uint32_t)(len - sizeof(*bss))))
 		return;
 
 	RSU_DPRINTF(sc, RSU_DEBUG_SCAN,
 	    "%s: found BSS %s: len=%d chan=%d inframode=%d "
 	    "networktype=%d privacy=%d, RSSI=%d\n",
 	    __func__,
-	    ether_sprintf(bss->macaddr), le32toh(bss->len),
+	    ether_sprintf(bss->macaddr), ieslen,
 	    le32toh(bss->config.dsconfig), le32toh(bss->inframode),
 	    le32toh(bss->networktype), le32toh(bss->privacy),
 	    le32toh(bss->rssi));
 
 	/* Build a fake beacon frame to let net80211 do all the parsing. */
 	/* XXX TODO: just call the new scan API methods! */
-	pktlen = sizeof(*wh) + le32toh(bss->ieslen);
-	if (__predict_false(pktlen > MCLBYTES))
+	if (__predict_false(ieslen > (size_t)(MCLBYTES - sizeof(*wh))))
 		return;
+	pktlen = sizeof(*wh) + ieslen;
 	m = m_get2(pktlen, M_NOWAIT, MT_DATA, M_PKTHDR);
 	if (__predict_false(m == NULL))
 		return;
@@ -2078,7 +2083,7 @@ rsu_event_survey(struct rsu_softc *sc, uint8_t *buf, int len)
 	IEEE80211_ADDR_COPY(wh->i_addr2, bss->macaddr);
 	IEEE80211_ADDR_COPY(wh->i_addr3, bss->macaddr);
 	*(uint16_t *)wh->i_seq = 0;
-	memcpy(&wh[1], (uint8_t *)&bss[1], le32toh(bss->ieslen));
+	memcpy(&wh[1], (uint8_t *)&bss[1], ieslen);
 
 	/* Finalize mbuf. */
 	m->m_pkthdr.len = m->m_len = pktlen;
@@ -3378,6 +3383,8 @@ rsu_fw_loadsection(struct rsu_softc *sc, const uint8_t *buf, int len)
 	return (0);
 }
 
+CTASSERT(sizeof(size_t) >= sizeof(uint32_t));
+
 static int
 rsu_load_firmware(struct rsu_softc *sc)
 {
@@ -3385,7 +3392,7 @@ rsu_load_firmware(struct rsu_softc *sc)
 	struct r92s_fw_priv *dmem;
 	struct ieee80211com *ic = &sc->sc_ic;
 	const uint8_t *imem, *emem;
-	int imemsz, ememsz;
+	uint32_t imemsz, ememsz;
 	const struct firmware *fw;
 	size_t size;
 	uint32_t reg;
@@ -3437,7 +3444,8 @@ rsu_load_firmware(struct rsu_softc *sc)
 	imemsz = le32toh(hdr->imemsz);
 	ememsz = le32toh(hdr->sramsz);
 	/* Check that all FW sections fit in image. */
-	if (size < sizeof(*hdr) + imemsz + ememsz) {
+	if (imemsz > (size_t)(size - sizeof(*hdr)) ||
+	    ememsz > (size_t)(size - sizeof(*hdr) - imemsz)) {
 		device_printf(sc->sc_dev, "firmware too short\n");
 		error = EINVAL;
 		goto fail;
diff --git a/freebsd/sys/dev/usb/wlan/if_zyd.c b/freebsd/sys/dev/usb/wlan/if_zyd.c
index 3ae5616f..e4243d8f 100644
--- a/freebsd/sys/dev/usb/wlan/if_zyd.c
+++ b/freebsd/sys/dev/usb/wlan/if_zyd.c
@@ -650,11 +650,12 @@ zyd_intr_read_callback(struct usb_xfer *xfer, usb_error_t error)
 		{
 			struct zyd_notif_retry *retry =
 			    (struct zyd_notif_retry *)cmd->data;
+			uint16_t count = le16toh(retry->count);
 
 			DPRINTF(sc, ZYD_DEBUG_TX_PROC,
 			    "retry intr: rate=0x%x addr=%s count=%d (0x%x)\n",
 			    le16toh(retry->rate), ether_sprintf(retry->macaddr),
-			    le16toh(retry->count)&0xff, le16toh(retry->count));
+			    count & 0xff, count);
 
 			/*
 			 * Find the node to which the packet was sent and
@@ -666,13 +667,12 @@ zyd_intr_read_callback(struct usb_xfer *xfer, usb_error_t error)
 			if (ni != NULL) {
 				struct ieee80211_ratectl_tx_status *txs =
 				    &sc->sc_txs;
-				int retrycnt =
-				    (int)(le16toh(retry->count) & 0xff);
+				int retrycnt = count & 0xff;
 
 				txs->flags =
 				    IEEE80211_RATECTL_STATUS_LONG_RETRY;
 				txs->long_retries = retrycnt;
-				if (le16toh(retry->count) & 0x100) {
+				if (count & 0x100) {
 					txs->status =
 					    IEEE80211_RATECTL_TX_FAIL_LONG;
 				} else {
@@ -684,7 +684,7 @@ zyd_intr_read_callback(struct usb_xfer *xfer, usb_error_t error)
 				ieee80211_ratectl_tx_complete(ni, txs);
 				ieee80211_free_node(ni);
 			}
-			if (le16toh(retry->count) & 0x100)
+			if (count & 0x100)
 				/* too many retries */
 				if_inc_counter(vap->iv_ifp, IFCOUNTER_OERRORS,
 				    1);