Update to FreeBSD head 2017-08-01

Git mirror commit f5002f5e5f78cae9f0269d812dc0aedb0339312c.

Update #3472.

1 files changed, 976 insertions, 45 deletions

diff --git a/freebsd/sys/dev/sdhci/sdhci.c b/freebsd/sys/dev/sdhci/sdhci.c
index c87199a8..f1616a6e 100644
--- a/freebsd/sys/dev/sdhci/sdhci.c
+++ b/freebsd/sys/dev/sdhci/sdhci.c
@@ -2,6 +2,7 @@
 
 /*-
  * Copyright (c) 2008 Alexander Motin <mav@FreeBSD.org>
+ * Copyright (c) 2017 Marius Strobl <marius@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -34,7 +35,9 @@ __FBSDID("$FreeBSD$");
 #include <sys/callout.h>
 #include <sys/conf.h>
 #include <sys/kernel.h>
+#include <sys/kobj.h>
 #include <sys/lock.h>
+#include <sys/malloc.h>
 #include <sys/module.h>
 #include <sys/mutex.h>
 #include <rtems/bsd/sys/resource.h>
@@ -50,13 +53,22 @@ __FBSDID("$FreeBSD$");
 #include <dev/mmc/mmcreg.h>
 #include <dev/mmc/mmcbrvar.h>
 
+#include <dev/sdhci/sdhci.h>
+
+#include <cam/cam.h>
+#include <cam/cam_ccb.h>
+#include <cam/cam_debug.h>
+#include <cam/cam_sim.h>
+#include <cam/cam_xpt_sim.h>
+
 #include <rtems/bsd/local/mmcbr_if.h>
-#include "sdhci.h"
 #include <rtems/bsd/local/sdhci_if.h>
 
+#include <rtems/bsd/local/opt_mmccam.h>
+
 SYSCTL_NODE(_hw, OID_AUTO, sdhci, CTLFLAG_RD, 0, "sdhci driver");
 
-static int sdhci_debug;
+static int sdhci_debug = 0;
 SYSCTL_INT(_hw_sdhci, OID_AUTO, debug, CTLFLAG_RWTUN, &sdhci_debug, 0,
     "Debug level");
 u_int sdhci_quirk_clear = 0;
@@ -78,17 +90,30 @@ SYSCTL_INT(_hw_sdhci, OID_AUTO, quirk_set, CTLFLAG_RWTUN, &sdhci_quirk_set, 0,
 #define	WR_MULTI_4(slot, off, ptr, count)	\
     SDHCI_WRITE_MULTI_4((slot)->bus, (slot), (off), (ptr), (count))
 
+static void sdhci_card_poll(void *arg);
+static void sdhci_card_task(void *arg, int pending);
+static int sdhci_exec_tuning(struct sdhci_slot *slot, bool reset);
+static void sdhci_req_wakeup(struct mmc_request *req);
+static void sdhci_retune(void *arg);
 static void sdhci_set_clock(struct sdhci_slot *slot, uint32_t clock);
 static void sdhci_start(struct sdhci_slot *slot);
 static void sdhci_start_data(struct sdhci_slot *slot, struct mmc_data *data);
 
-static void sdhci_card_poll(void *);
-static void sdhci_card_task(void *, int);
+#ifdef MMCCAM
+/* CAM-related */
+int sdhci_cam_get_possible_host_clock(struct sdhci_slot *slot, int proposed_clock);
+static int sdhci_cam_update_ios(struct sdhci_slot *slot);
+static int sdhci_cam_request(struct sdhci_slot *slot, union ccb *ccb);
+static void sdhci_cam_action(struct cam_sim *sim, union ccb *ccb);
+static void sdhci_cam_poll(struct cam_sim *sim);
+static int sdhci_cam_settran_settings(struct sdhci_slot *slot, union ccb *ccb);
+#endif
 
 /* helper routines */
 static void sdhci_dumpregs(struct sdhci_slot *slot);
 static int slot_printf(struct sdhci_slot *slot, const char * fmt, ...)
     __printflike(2, 3);
+static uint32_t sdhci_tuning_intmask(struct sdhci_slot *slot);
 
 #define	SDHCI_LOCK(_slot)		mtx_lock(&(_slot)->mtx)
 #define	SDHCI_UNLOCK(_slot)		mtx_unlock(&(_slot)->mtx)
@@ -169,13 +194,13 @@ sdhci_dumpregs(struct sdhci_slot *slot)
 	    RD1(slot, SDHCI_TIMEOUT_CONTROL), RD4(slot, SDHCI_INT_STATUS));
 	slot_printf(slot, "Int enab: 0x%08x | Sig enab: 0x%08x\n",
 	    RD4(slot, SDHCI_INT_ENABLE), RD4(slot, SDHCI_SIGNAL_ENABLE));
-	slot_printf(slot, "AC12 err: 0x%08x | Host ctl2: 0x%08x\n",
+	slot_printf(slot, "AC12 err: 0x%08x | Host ctl2:0x%08x\n",
 	    RD2(slot, SDHCI_ACMD12_ERR), RD2(slot, SDHCI_HOST_CONTROL2));
 	slot_printf(slot, "Caps:     0x%08x | Caps2:    0x%08x\n",
 	    RD4(slot, SDHCI_CAPABILITIES), RD4(slot, SDHCI_CAPABILITIES2));
 	slot_printf(slot, "Max curr: 0x%08x | ADMA err: 0x%08x\n",
 	    RD4(slot, SDHCI_MAX_CURRENT), RD1(slot, SDHCI_ADMA_ERR));
-	slot_printf(slot, "ADMA addr: 0x%08x | Slot int: 0x%08x\n",
+	slot_printf(slot, "ADMA addr:0x%08x | Slot int: 0x%08x\n",
 	    RD4(slot, SDHCI_ADMA_ADDRESS_LO), RD2(slot, SDHCI_SLOT_INT_STATUS));
 
 	slot_printf(slot,
@@ -242,6 +267,21 @@ sdhci_reset(struct sdhci_slot *slot, uint8_t mask)
 	}
 }
 
+static uint32_t
+sdhci_tuning_intmask(struct sdhci_slot *slot)
+{
+	uint32_t intmask;
+
+	intmask = 0;
+	if (slot->opt & SDHCI_TUNING_SUPPORTED) {
+		intmask |= SDHCI_INT_TUNEERR;
+		if (slot->retune_mode == SDHCI_RETUNE_MODE_2 ||
+		    slot->retune_mode == SDHCI_RETUNE_MODE_3)
+			intmask |= SDHCI_INT_RETUNE;
+	}
+	return (intmask);
+}
+
 static void
 sdhci_init(struct sdhci_slot *slot)
 {
@@ -261,7 +301,7 @@ sdhci_init(struct sdhci_slot *slot)
 		slot->intmask |= SDHCI_INT_CARD_REMOVE | SDHCI_INT_CARD_INSERT;
 	}
 
-	WR4(slot, SDHCI_INT_ENABLE, slot->intmask);
+	WR4(slot, SDHCI_INT_ENABLE, slot->intmask | sdhci_tuning_intmask(slot));
 	WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask);
 }
 
@@ -368,6 +408,7 @@ sdhci_set_clock(struct sdhci_slot *slot, uint32_t clock)
 static void
 sdhci_set_power(struct sdhci_slot *slot, u_char power)
 {
+	int i;
 	uint8_t pwr;
 
 	if (slot->power == power)
@@ -396,9 +437,20 @@ sdhci_set_power(struct sdhci_slot *slot, u_char power)
 		break;
 	}
 	WR1(slot, SDHCI_POWER_CONTROL, pwr);
-	/* Turn on the power. */
+	/*
+	 * Turn on VDD1 power.  Note that at least some Intel controllers can
+	 * fail to enable bus power on the first try after transiting from D3
+	 * to D0, so we give them up to 2 ms.
+	 */
 	pwr |= SDHCI_POWER_ON;
-	WR1(slot, SDHCI_POWER_CONTROL, pwr);
+	for (i = 0; i < 20; i++) {
+		WR1(slot, SDHCI_POWER_CONTROL, pwr);
+		if (RD1(slot, SDHCI_POWER_CONTROL) & SDHCI_POWER_ON)
+			break;
+		DELAY(100);
+	}
+	if (!(RD1(slot, SDHCI_POWER_CONTROL) & SDHCI_POWER_ON))
+		slot_printf(slot, "Bus power failed to enable");
 
 	if (slot->quirks & SDHCI_QUIRK_INTEL_POWER_UP_RESET) {
 		WR1(slot, SDHCI_POWER_CONTROL, pwr | 0x10);
@@ -521,25 +573,93 @@ sdhci_card_task(void *arg, int pending __unused)
 
 	SDHCI_LOCK(slot);
 	if (SDHCI_GET_CARD_PRESENT(slot->bus, slot)) {
+#ifdef MMCCAM
+		if (slot->card_present == 0) {
+#else
 		if (slot->dev == NULL) {
+#endif
 			/* If card is present - attach mmc bus. */
 			if (bootverbose || sdhci_debug)
 				slot_printf(slot, "Card inserted\n");
-			slot->dev = device_add_child(slot->bus, "mmc", -1);
-			device_set_ivars(slot->dev, slot);
+#ifdef MMCCAM
+			slot->card_present = 1;
+			union ccb *ccb;
+			uint32_t pathid;
+			pathid = cam_sim_path(slot->sim);
+			ccb = xpt_alloc_ccb_nowait();
+			if (ccb == NULL) {
+				slot_printf(slot, "Unable to alloc CCB for rescan\n");
+				SDHCI_UNLOCK(slot);
+				return;
+			}
+
+			/*
+			 * We create a rescan request for BUS:0:0, since the card
+			 * will be at lun 0.
+			 */
+			if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid,
+					    /* target */ 0, /* lun */ 0) != CAM_REQ_CMP) {
+				slot_printf(slot, "Unable to create path for rescan\n");
+				SDHCI_UNLOCK(slot);
+				xpt_free_ccb(ccb);
+				return;
+			}
+			SDHCI_UNLOCK(slot);
+			xpt_rescan(ccb);
+#else
+			d = slot->dev = device_add_child(slot->bus, "mmc", -1);
 			SDHCI_UNLOCK(slot);
-			device_probe_and_attach(slot->dev);
+			if (d) {
+				device_set_ivars(d, slot);
+				(void)device_probe_and_attach(d);
+			}
+#endif
 		} else
 			SDHCI_UNLOCK(slot);
 	} else {
+#ifdef MMCCAM
+		if (slot->card_present == 1) {
+#else
 		if (slot->dev != NULL) {
+#endif
 			/* If no card present - detach mmc bus. */
 			if (bootverbose || sdhci_debug)
 				slot_printf(slot, "Card removed\n");
 			d = slot->dev;
 			slot->dev = NULL;
+#ifdef MMCCAM
+			slot->card_present = 0;
+			union ccb *ccb;
+			uint32_t pathid;
+			pathid = cam_sim_path(slot->sim);
+			ccb = xpt_alloc_ccb_nowait();
+			if (ccb == NULL) {
+				slot_printf(slot, "Unable to alloc CCB for rescan\n");
+				SDHCI_UNLOCK(slot);
+				return;
+			}
+
+			/*
+			 * We create a rescan request for BUS:0:0, since the card
+			 * will be at lun 0.
+			 */
+			if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid,
+					    /* target */ 0, /* lun */ 0) != CAM_REQ_CMP) {
+				slot_printf(slot, "Unable to create path for rescan\n");
+				SDHCI_UNLOCK(slot);
+				xpt_free_ccb(ccb);
+				return;
+			}
 			SDHCI_UNLOCK(slot);
+			xpt_rescan(ccb);
+#else
+			slot->intmask &= ~sdhci_tuning_intmask(slot);
+			WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask);
+			slot->opt &= ~SDHCI_TUNING_ENABLED;
+			SDHCI_UNLOCK(slot);
+			callout_drain(&slot->retune_callout);
 			device_delete_child(slot->bus, d);
+#endif
 		} else
 			SDHCI_UNLOCK(slot);
 	}
@@ -561,7 +681,11 @@ sdhci_handle_card_present_locked(struct sdhci_slot *slot, bool is_present)
 	 * because once power is removed, a full card re-init is needed, and
 	 * that happens by deleting and recreating the child device.
 	 */
+#ifdef MMCCAM
+	was_present = slot->card_present;
+#else
 	was_present = slot->dev != NULL;
+#endif
 	if (!was_present && is_present) {
 		taskqueue_enqueue_timeout(taskqueue_swi_giant,
 		    &slot->card_delayed_task, -SDHCI_INSERT_DELAY_TICKS);
@@ -593,10 +717,13 @@ sdhci_card_poll(void *arg)
 int
 sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
 {
+	kobjop_desc_t kobj_desc;
+	kobj_method_t *kobj_method;
 	uint32_t caps, caps2, freq, host_caps;
 	int err;
 
 	SDHCI_LOCK_INIT(slot);
+
 	slot->num = num;
 	slot->bus = dev;
 
@@ -617,6 +744,7 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
 	    BUS_DMA_NOWAIT, &slot->dmamap);
 	if (err != 0) {
 		device_printf(dev, "Can't alloc DMA memory\n");
+		bus_dma_tag_destroy(slot->dmatag);
 		SDHCI_LOCK_DESTROY(slot);
 		return (err);
 	}
@@ -626,6 +754,8 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
 	    sdhci_getaddr, &slot->paddr, 0);
 	if (err != 0 || slot->paddr == 0) {
 		device_printf(dev, "Can't load DMA memory\n");
+		bus_dmamem_free(slot->dmatag, slot->dmamem, slot->dmamap);
+		bus_dma_tag_destroy(slot->dmatag);
 		SDHCI_LOCK_DESTROY(slot);
 		if (err)
 			return (err);
@@ -633,8 +763,6 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
 			return (EFAULT);
 	}
 
-	/* Initialize slot. */
-	sdhci_init(slot);
 	slot->version = (RD2(slot, SDHCI_HOST_VERSION)
 		>> SDHCI_SPEC_VER_SHIFT) & SDHCI_SPEC_VER_MASK;
 	if (slot->quirks & SDHCI_QUIRK_MISSING_CAPS) {
@@ -647,6 +775,22 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
 		else
 			caps2 = 0;
 	}
+	if (slot->version >= SDHCI_SPEC_300) {
+		if ((caps & SDHCI_SLOTTYPE_MASK) != SDHCI_SLOTTYPE_REMOVABLE &&
+		    (caps & SDHCI_SLOTTYPE_MASK) != SDHCI_SLOTTYPE_EMBEDDED) {
+			device_printf(dev,
+			    "Driver doesn't support shared bus slots\n");
+			bus_dmamap_unload(slot->dmatag, slot->dmamap);
+			bus_dmamem_free(slot->dmatag, slot->dmamem,
+			    slot->dmamap);
+			bus_dma_tag_destroy(slot->dmatag);
+			SDHCI_LOCK_DESTROY(slot);
+			return (ENXIO);
+		} else if ((caps & SDHCI_SLOTTYPE_MASK) ==
+		    SDHCI_SLOTTYPE_EMBEDDED) {
+			slot->opt |= SDHCI_SLOT_EMBEDDED | SDHCI_NON_REMOVABLE;
+		}
+	}
 	/* Calculate base clock frequency. */
 	if (slot->version >= SDHCI_SPEC_300)
 		freq = (caps & SDHCI_CLOCK_V3_BASE_MASK) >>
@@ -696,12 +840,14 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
 	    slot->host.host_ocr |= MMC_OCR_320_330 | MMC_OCR_330_340;
 	if (caps & SDHCI_CAN_VDD_300)
 	    slot->host.host_ocr |= MMC_OCR_290_300 | MMC_OCR_300_310;
-	if (caps & SDHCI_CAN_VDD_180)
+	/* 1.8V VDD is not supposed to be used for removable cards. */
+	if ((caps & SDHCI_CAN_VDD_180) && (slot->opt & SDHCI_SLOT_EMBEDDED))
 	    slot->host.host_ocr |= MMC_OCR_LOW_VOLTAGE;
 	if (slot->host.host_ocr == 0) {
 		device_printf(dev, "Hardware doesn't report any "
 		    "support voltages.\n");
 	}
+
 	host_caps = MMC_CAP_4_BIT_DATA;
 	if (caps & SDHCI_CAN_DO_8BITBUS)
 		host_caps |= MMC_CAP_8_BIT_DATA;
@@ -711,6 +857,8 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
 		host_caps |= MMC_CAP_BOOT_NOACC;
 	if (slot->quirks & SDHCI_QUIRK_WAIT_WHILE_BUSY)
 		host_caps |= MMC_CAP_WAIT_WHILE_BUSY;
+
+	/* Determine supported UHS-I and eMMC modes. */
 	if (caps2 & (SDHCI_CAN_SDR50 | SDHCI_CAN_SDR104 | SDHCI_CAN_DDR50))
 		host_caps |= MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25;
 	if (caps2 & SDHCI_CAN_SDR104) {
@@ -727,12 +875,91 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
 	if (slot->quirks & SDHCI_QUIRK_CAPS_BIT63_FOR_MMC_HS400 &&
 	    caps2 & SDHCI_CAN_MMC_HS400)
 		host_caps |= MMC_CAP_MMC_HS400;
+
+	/*
+	 * Disable UHS-I and eMMC modes if the set_uhs_timing method is the
+	 * default NULL implementation.
+	 */
+	kobj_desc = &sdhci_set_uhs_timing_desc;
+	kobj_method = kobj_lookup_method(((kobj_t)dev)->ops->cls, NULL,
+	    kobj_desc);
+	if (kobj_method == &kobj_desc->deflt)
+		host_caps &= ~(MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25 |
+		    MMC_CAP_UHS_SDR50 | MMC_CAP_UHS_DDR50 | MMC_CAP_UHS_SDR104 |
+		    MMC_CAP_MMC_DDR52 | MMC_CAP_MMC_HS200 | MMC_CAP_MMC_HS400);
+
+#define	SDHCI_CAP_MODES_TUNING(caps2)					\
+    (((caps2) & SDHCI_TUNE_SDR50 ? MMC_CAP_UHS_SDR50 : 0) |		\
+    MMC_CAP_UHS_DDR50 | MMC_CAP_UHS_SDR104 | MMC_CAP_MMC_HS200 |	\
+    MMC_CAP_MMC_HS400)
+
+	/*
+	 * Disable UHS-I and eMMC modes that require (re-)tuning if either
+	 * the tune or re-tune method is the default NULL implementation.
+	 */
+	kobj_desc = &mmcbr_tune_desc;
+	kobj_method = kobj_lookup_method(((kobj_t)dev)->ops->cls, NULL,
+	    kobj_desc);
+	if (kobj_method == &kobj_desc->deflt)
+		goto no_tuning;
+	kobj_desc = &mmcbr_retune_desc;
+	kobj_method = kobj_lookup_method(((kobj_t)dev)->ops->cls, NULL,
+	    kobj_desc);
+	if (kobj_method == &kobj_desc->deflt) {
+no_tuning:
+		host_caps &= ~(SDHCI_CAP_MODES_TUNING(caps2));
+	}
+
+	/* Allocate tuning structures and determine tuning parameters. */
+	if (host_caps & SDHCI_CAP_MODES_TUNING(caps2)) {
+		slot->opt |= SDHCI_TUNING_SUPPORTED;
+		slot->tune_req = malloc(sizeof(*slot->tune_req), M_DEVBUF,
+		    M_WAITOK);
+		slot->tune_cmd = malloc(sizeof(*slot->tune_cmd), M_DEVBUF,
+		    M_WAITOK);
+		slot->tune_data = malloc(sizeof(*slot->tune_data), M_DEVBUF,
+		    M_WAITOK);
+		if (caps2 & SDHCI_TUNE_SDR50)
+			slot->opt |= SDHCI_SDR50_NEEDS_TUNING;
+		slot->retune_mode = (caps2 & SDHCI_RETUNE_MODES_MASK) >>
+		    SDHCI_RETUNE_MODES_SHIFT;
+		if (slot->retune_mode == SDHCI_RETUNE_MODE_1) {
+			slot->retune_count = (caps2 & SDHCI_RETUNE_CNT_MASK) >>
+			    SDHCI_RETUNE_CNT_SHIFT;
+			if (slot->retune_count > 0xb) {
+				device_printf(dev, "Unknown re-tuning count "
+				    "%x, using 1 sec\n", slot->retune_count);
+				slot->retune_count = 1;
+			} else if (slot->retune_count != 0)
+				slot->retune_count =
+				    1 << (slot->retune_count - 1);
+		}
+	}
+
+#undef SDHCI_CAP_MODES_TUNING
+
+	/* Determine supported VCCQ signaling levels. */
 	host_caps |= MMC_CAP_SIGNALING_330;
 	if (host_caps & (MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25 |
-	    MMC_CAP_UHS_SDR50 | MMC_CAP_UHS_SDR104 | MMC_CAP_UHS_DDR50 |
+	    MMC_CAP_UHS_SDR50 | MMC_CAP_UHS_DDR50 | MMC_CAP_UHS_SDR104 |
 	    MMC_CAP_MMC_DDR52_180 | MMC_CAP_MMC_HS200_180 |
 	    MMC_CAP_MMC_HS400_180))
-		host_caps |= MMC_CAP_SIGNALING_180;
+		host_caps |= MMC_CAP_SIGNALING_120 | MMC_CAP_SIGNALING_180;
+
+	/*
+	 * Disable 1.2 V and 1.8 V signaling if the switch_vccq method is the
+	 * default NULL implementation.  Disable 1.2 V support if it's the
+	 * generic SDHCI implementation.
+	 */
+	kobj_desc = &mmcbr_switch_vccq_desc;
+	kobj_method = kobj_lookup_method(((kobj_t)dev)->ops->cls, NULL,
+	    kobj_desc);
+	if (kobj_method == &kobj_desc->deflt)
+		host_caps &= ~(MMC_CAP_SIGNALING_120 | MMC_CAP_SIGNALING_180);
+	else if (kobj_method->func == (kobjop_t)sdhci_generic_switch_vccq)
+		host_caps &= ~MMC_CAP_SIGNALING_120;
+
+	/* Determine supported driver types (type B is always mandatory). */
 	if (caps2 & SDHCI_CAN_DRIVE_TYPE_A)
 		host_caps |= MMC_CAP_DRIVER_TYPE_A;
 	if (caps2 & SDHCI_CAN_DRIVE_TYPE_C)
@@ -761,20 +988,24 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
 
 	if (bootverbose || sdhci_debug) {
 		slot_printf(slot,
-		    "%uMHz%s %s VDD:%s%s%s VCCQ: 3.3V%s%s DRV: B%s%s%s %s\n",
+		    "%uMHz%s %s VDD:%s%s%s VCCQ: 3.3V%s%s DRV: B%s%s%s %s %s\n",
 		    slot->max_clk / 1000000,
 		    (caps & SDHCI_CAN_DO_HISPD) ? " HS" : "",
 		    (host_caps & MMC_CAP_8_BIT_DATA) ? "8bits" :
 			((host_caps & MMC_CAP_4_BIT_DATA) ? "4bits" : "1bit"),
 		    (caps & SDHCI_CAN_VDD_330) ? " 3.3V" : "",
 		    (caps & SDHCI_CAN_VDD_300) ? " 3.0V" : "",
-		    (caps & SDHCI_CAN_VDD_180) ? " 1.8V" : "",
+		    ((caps & SDHCI_CAN_VDD_180) &&
+		    (slot->opt & SDHCI_SLOT_EMBEDDED)) ? " 1.8V" : "",
 		    (host_caps & MMC_CAP_SIGNALING_180) ? " 1.8V" : "",
 		    (host_caps & MMC_CAP_SIGNALING_120) ? " 1.2V" : "",
-		    (caps2 & SDHCI_CAN_DRIVE_TYPE_A) ? "A" : "",
-		    (caps2 & SDHCI_CAN_DRIVE_TYPE_C) ? "C" : "",
-		    (caps2 & SDHCI_CAN_DRIVE_TYPE_D) ? "D" : "",
-		    (slot->opt & SDHCI_HAVE_DMA) ? "DMA" : "PIO");
+		    (host_caps & MMC_CAP_DRIVER_TYPE_A) ? "A" : "",
+		    (host_caps & MMC_CAP_DRIVER_TYPE_C) ? "C" : "",
+		    (host_caps & MMC_CAP_DRIVER_TYPE_D) ? "D" : "",
+		    (slot->opt & SDHCI_HAVE_DMA) ? "DMA" : "PIO",
+		    (slot->opt & SDHCI_SLOT_EMBEDDED) ? "embedded" :
+		    (slot->opt & SDHCI_NON_REMOVABLE) ? "non-removable" :
+		    "removable");
 		if (host_caps & (MMC_CAP_MMC_DDR52 | MMC_CAP_MMC_HS200 |
 		    MMC_CAP_MMC_HS400 | MMC_CAP_MMC_ENH_STROBE))
 			slot_printf(slot, "eMMC:%s%s%s%s\n",
@@ -793,6 +1024,9 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
 			    (host_caps & MMC_CAP_UHS_SDR50) ? " SDR50" : "",
 			    (host_caps & MMC_CAP_UHS_SDR104) ? " SDR104" : "",
 			    (host_caps & MMC_CAP_UHS_DDR50) ? " DDR50" : "");
+		if (slot->opt & SDHCI_TUNING_SUPPORTED)
+			slot_printf(slot, "Re-tuning count %d secs, mode %d\n",
+			    slot->retune_count, slot->retune_mode + 1);
 		sdhci_dumpregs(slot);
 	}
 
@@ -806,6 +1040,7 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
 		sdhci_card_task, slot);
 	callout_init(&slot->card_poll_callout, 1);
 	callout_init_mtx(&slot->timeout_callout, &slot->mtx, 0);
+	callout_init_mtx(&slot->retune_callout, &slot->mtx, 0);
 
 	if ((slot->quirks & SDHCI_QUIRK_POLL_CARD_PRESENT) &&
 	    !(slot->opt & SDHCI_NON_REMOVABLE)) {
@@ -813,6 +1048,8 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
 		    SDHCI_CARD_PRESENT_TICKS, sdhci_card_poll, slot);
 	}
 
+	sdhci_init(slot);
+
 	return (0);
 }
 
@@ -830,6 +1067,7 @@ sdhci_cleanup_slot(struct sdhci_slot *slot)
 
 	callout_drain(&slot->timeout_callout);
 	callout_drain(&slot->card_poll_callout);
+	callout_drain(&slot->retune_callout);
 	taskqueue_drain(taskqueue_swi_giant, &slot->card_task);
 	taskqueue_drain_timeout(taskqueue_swi_giant, &slot->card_delayed_task);
 
@@ -846,6 +1084,11 @@ sdhci_cleanup_slot(struct sdhci_slot *slot)
 	bus_dmamap_unload(slot->dmatag, slot->dmamap);
 	bus_dmamem_free(slot->dmatag, slot->dmamem, slot->dmamap);
 	bus_dma_tag_destroy(slot->dmatag);
+	if (slot->opt & SDHCI_TUNING_SUPPORTED) {
+		free(slot->tune_req, M_DEVBUF);
+		free(slot->tune_cmd, M_DEVBUF);
+		free(slot->tune_data, M_DEVBUF);
+	}
 
 	SDHCI_LOCK_DESTROY(slot);
 
@@ -856,7 +1099,16 @@ int
 sdhci_generic_suspend(struct sdhci_slot *slot)
 {
 
+	/*
+	 * We expect the MMC layer to issue initial tuning after resume.
+	 * Otherwise, we'd need to indicate re-tuning including circuit reset
+	 * being required at least for re-tuning modes 1 and 2 ourselves.
+	 */
+	callout_drain(&slot->retune_callout);
+	SDHCI_LOCK(slot);
+	slot->opt &= ~SDHCI_TUNING_ENABLED;
 	sdhci_reset(slot, SDHCI_RESET_ALL);
+	SDHCI_UNLOCK(slot);
 
 	return (0);
 }
@@ -865,7 +1117,9 @@ int
 sdhci_generic_resume(struct sdhci_slot *slot)
 {
 
+	SDHCI_LOCK(slot);
 	sdhci_init(slot);
+	SDHCI_UNLOCK(slot);
 
 	return (0);
 }
@@ -899,15 +1153,18 @@ sdhci_generic_set_uhs_timing(device_t brdev __unused, struct sdhci_slot *slot)
 	if (slot->version < SDHCI_SPEC_300)
 		return;
 
+	SDHCI_ASSERT_LOCKED(slot);
 	ios = &slot->host.ios;
 	sdhci_set_clock(slot, 0);
 	hostctrl2 = RD2(slot, SDHCI_HOST_CONTROL2);
 	hostctrl2 &= ~SDHCI_CTRL2_UHS_MASK;
-	if (ios->timing == bus_timing_mmc_hs400 ||
-	    ios->timing == bus_timing_mmc_hs400es)
-		hostctrl2 |= SDHCI_CTRL2_MMC_HS400;
-	else if (ios->clock > SD_SDR50_MAX)
-		hostctrl2 |= SDHCI_CTRL2_UHS_SDR104;
+	if (ios->clock > SD_SDR50_MAX) {
+		if (ios->timing == bus_timing_mmc_hs400 ||
+		    ios->timing == bus_timing_mmc_hs400es)
+			hostctrl2 |= SDHCI_CTRL2_MMC_HS400;
+		else
+			hostctrl2 |= SDHCI_CTRL2_UHS_SDR104;
+	}
 	else if (ios->clock > SD_SDR25_MAX)
 		hostctrl2 |= SDHCI_CTRL2_UHS_SDR50;
 	else if (ios->clock > SD_SDR12_MAX) {
@@ -1019,6 +1276,222 @@ done:
 	return (err);
 }
 
+int
+sdhci_generic_tune(device_t brdev __unused, device_t reqdev, bool hs400)
+{
+	struct sdhci_slot *slot = device_get_ivars(reqdev);
+	struct mmc_ios *ios = &slot->host.ios;
+	struct mmc_command *tune_cmd;
+	struct mmc_data *tune_data;
+	uint32_t opcode;
+	int err;
+
+	if (!(slot->opt & SDHCI_TUNING_SUPPORTED))
+		return (0);
+
+	slot->retune_ticks = slot->retune_count * hz;
+	opcode = MMC_SEND_TUNING_BLOCK;
+	SDHCI_LOCK(slot);
+	switch (ios->timing) {
+	case bus_timing_mmc_hs400:
+		slot_printf(slot, "HS400 must be tuned in HS200 mode\n");
+		SDHCI_UNLOCK(slot);
+		return (EINVAL);
+	case bus_timing_mmc_hs200:
+		/*
+		 * In HS400 mode, controllers use the data strobe line to
+		 * latch data from the devices so periodic re-tuning isn't
+		 * expected to be required.
+		 */
+		if (hs400)
+			slot->retune_ticks = 0;
+		opcode = MMC_SEND_TUNING_BLOCK_HS200;
+		break;
+	case bus_timing_uhs_ddr50:
+	case bus_timing_uhs_sdr104:
+		break;
+	case bus_timing_uhs_sdr50:
+		if (slot->opt & SDHCI_SDR50_NEEDS_TUNING)
+			break;
+		/* FALLTHROUGH */
+	default:
+		SDHCI_UNLOCK(slot);
+		return (0);
+	}
+
+	tune_cmd = slot->tune_cmd;
+	memset(tune_cmd, 0, sizeof(*tune_cmd));
+	tune_cmd->opcode = opcode;
+	tune_cmd->flags = MMC_RSP_R1 | MMC_CMD_ADTC;
+	tune_data = tune_cmd->data = slot->tune_data;
+	memset(tune_data, 0, sizeof(*tune_data));
+	tune_data->len = (opcode == MMC_SEND_TUNING_BLOCK_HS200 &&
+	    ios->bus_width == bus_width_8) ? MMC_TUNING_LEN_HS200 :
+	    MMC_TUNING_LEN;
+	tune_data->flags = MMC_DATA_READ;
+	tune_data->mrq = tune_cmd->mrq = slot->tune_req;
+
+	slot->opt &= ~SDHCI_TUNING_ENABLED;
+	err = sdhci_exec_tuning(slot, true);
+	if (err == 0) {
+		slot->opt |= SDHCI_TUNING_ENABLED;
+		slot->intmask |= sdhci_tuning_intmask(slot);
+		WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask);
+		if (slot->retune_ticks) {
+			callout_reset(&slot->retune_callout, slot->retune_ticks,
+			    sdhci_retune, slot);
+		}
+	}
+	SDHCI_UNLOCK(slot);
+	return (err);
+}
+
+int
+sdhci_generic_retune(device_t brdev __unused, device_t reqdev, bool reset)
+{
+	struct sdhci_slot *slot = device_get_ivars(reqdev);
+	int err;
+
+	if (!(slot->opt & SDHCI_TUNING_ENABLED))
+		return (0);
+
+	/* HS400 must be tuned in HS200 mode. */
+	if (slot->host.ios.timing == bus_timing_mmc_hs400)
+		return (EINVAL);
+
+	SDHCI_LOCK(slot);
+	err = sdhci_exec_tuning(slot, reset);
+	/*
+	 * There are two ways sdhci_exec_tuning() can fail:
+	 * EBUSY should not actually happen when requests are only issued
+	 *	 with the host properly acquired, and
+	 * EIO   re-tuning failed (but it did work initially).
+	 *
+	 * In both cases, we should retry at later point if periodic re-tuning
+	 * is enabled.  Note that due to slot->retune_req not being cleared in
+	 * these failure cases, the MMC layer should trigger another attempt at
+	 * re-tuning with the next request anyway, though.
+	 */
+	if (slot->retune_ticks) {
+		callout_reset(&slot->retune_callout, slot->retune_ticks,
+		    sdhci_retune, slot);
+	}
+	SDHCI_UNLOCK(slot);
+	return (err);
+}
+
+static int
+sdhci_exec_tuning(struct sdhci_slot *slot, bool reset)
+{
+	struct mmc_request *tune_req;
+	struct mmc_command *tune_cmd;
+	int i;
+	uint32_t intmask;
+	uint16_t hostctrl2;
+	u_char opt;
+
+	SDHCI_ASSERT_LOCKED(slot);
+	if (slot->req != NULL)
+		return (EBUSY);
+
+	/* Tuning doesn't work with DMA enabled. */
+	opt = slot->opt;
+	slot->opt = opt & ~SDHCI_HAVE_DMA;
+
+	/*
+	 * Ensure that as documented, SDHCI_INT_DATA_AVAIL is the only
+	 * kind of interrupt we receive in response to a tuning request.
+	 */
+	intmask = slot->intmask;
+	slot->intmask = SDHCI_INT_DATA_AVAIL;
+	WR4(slot, SDHCI_SIGNAL_ENABLE, SDHCI_INT_DATA_AVAIL);
+
+	hostctrl2 = RD2(slot, SDHCI_HOST_CONTROL2);
+	if (reset)
+		hostctrl2 &= ~SDHCI_CTRL2_SAMPLING_CLOCK;
+	else
+		hostctrl2 |= SDHCI_CTRL2_SAMPLING_CLOCK;
+	WR2(slot, SDHCI_HOST_CONTROL2, hostctrl2 | SDHCI_CTRL2_EXEC_TUNING);
+
+	tune_req = slot->tune_req;
+	tune_cmd = slot->tune_cmd;
+	for (i = 0; i < MMC_TUNING_MAX; i++) {
+		memset(tune_req, 0, sizeof(*tune_req));
+#ifdef __rtems__
+		rtems_binary_semaphore_init(&tune_req->req_done,
+		    "sdhci_req_done");
+#endif /* __rtems__ */
+		tune_req->cmd = tune_cmd;
+		tune_req->done = sdhci_req_wakeup;
+		tune_req->done_data = slot;
+		slot->req = tune_req;
+		slot->flags = 0;
+		sdhci_start(slot);
+#ifndef __rtems__
+		while (!(tune_req->flags & MMC_REQ_DONE))
+			msleep(tune_req, &slot->mtx, 0, "sdhciet", 0);
+#else /* __rtems__ */
+		rtems_binary_semaphore_wait(&tune_req->req_done);
+		rtems_binary_semaphore_destroy(&tune_req->req_done);
+#endif /* __rtems__ */
+		if (!(tune_req->flags & MMC_TUNE_DONE))
+			break;
+		hostctrl2 = RD2(slot, SDHCI_HOST_CONTROL2);
+		if (!(hostctrl2 & SDHCI_CTRL2_EXEC_TUNING))
+			break;
+		if (tune_cmd->opcode == MMC_SEND_TUNING_BLOCK)
+			DELAY(1000);
+	}
+
+	slot->opt = opt;
+	slot->intmask = intmask;
+	WR4(slot, SDHCI_SIGNAL_ENABLE, intmask);
+
+	if ((hostctrl2 & (SDHCI_CTRL2_EXEC_TUNING |
+	    SDHCI_CTRL2_SAMPLING_CLOCK)) == SDHCI_CTRL2_SAMPLING_CLOCK) {
+		slot->retune_req = 0;
+		return (0);
+	}
+
+	slot_printf(slot, "Tuning failed, using fixed sampling clock\n");
+	WR2(slot, SDHCI_HOST_CONTROL2, hostctrl2 & ~(SDHCI_CTRL2_EXEC_TUNING |
+	    SDHCI_CTRL2_SAMPLING_CLOCK));
+	sdhci_reset(slot, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
+	return (EIO);
+}
+
+static void
+sdhci_retune(void *arg)
+{
+	struct sdhci_slot *slot = arg;
+
+	slot->retune_req |= SDHCI_RETUNE_REQ_NEEDED;
+}
+
+#ifdef MMCCAM
+static void
+sdhci_req_done(struct sdhci_slot *slot)
+{
+        union ccb *ccb;
+
+	if (__predict_false(sdhci_debug > 1))
+		slot_printf(slot, "%s\n", __func__);
+	if (slot->ccb != NULL && slot->curcmd != NULL) {
+		callout_stop(&slot->timeout_callout);
+                ccb = slot->ccb;
+                slot->ccb = NULL;
+		slot->curcmd = NULL;
+
+                /* Tell CAM the request is finished */
+                struct ccb_mmcio *mmcio;
+                mmcio = &ccb->mmcio;
+
+                ccb->ccb_h.status =
+                        (mmcio->cmd.error == 0 ? CAM_REQ_CMP : CAM_REQ_CMP_ERR);
+                xpt_done(ccb);
+	}
+}
+#else
 static void
 sdhci_req_done(struct sdhci_slot *slot)
 {
@@ -1032,6 +1505,21 @@ sdhci_req_done(struct sdhci_slot *slot)
 		req->done(req);
 	}
 }
+#endif
+
+static void
+sdhci_req_wakeup(struct mmc_request *req)
+{
+#ifndef __rtems__
+	struct sdhci_slot *slot;
+
+	slot = req->done_data;
+	req->flags |= MMC_REQ_DONE;
+	wakeup(req);
+#else /* __rtems__ */
+	rtems_binary_semaphore_post(&req->req_done);
+#endif /* __rtems__ */
+}
 
 static void
 sdhci_timeout(void *arg)
@@ -1039,13 +1527,13 @@ sdhci_timeout(void *arg)
 	struct sdhci_slot *slot = arg;
 
 	if (slot->curcmd != NULL) {
-		slot_printf(slot, " Controller timeout\n");
+		slot_printf(slot, "Controller timeout\n");
 		sdhci_dumpregs(slot);
 		sdhci_reset(slot, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
 		slot->curcmd->error = MMC_ERR_TIMEOUT;
 		sdhci_req_done(slot);
 	} else {
-		slot_printf(slot, " Spurious timeout - no active command\n");
+		slot_printf(slot, "Spurious timeout - no active command\n");
 	}
 }
 
@@ -1062,8 +1550,16 @@ sdhci_set_transfer_mode(struct sdhci_slot *slot, struct mmc_data *data)
 		mode |= SDHCI_TRNS_MULTI;
 	if (data->flags & MMC_DATA_READ)
 		mode |= SDHCI_TRNS_READ;
+#ifdef MMCCAM
+	struct ccb_mmcio *mmcio;
+	mmcio = &slot->ccb->mmcio;
+	if (mmcio->stop.opcode == MMC_STOP_TRANSMISSION
+	    && !(slot->quirks & SDHCI_QUIRK_BROKEN_AUTO_STOP))
+		mode |= SDHCI_TRNS_ACMD12;
+#else
 	if (slot->req->stop && !(slot->quirks & SDHCI_QUIRK_BROKEN_AUTO_STOP))
 		mode |= SDHCI_TRNS_ACMD12;
+#endif
 	if (slot->flags & SDHCI_USE_DMA)
 		mode |= SDHCI_TRNS_DMA;
 
@@ -1096,6 +1592,9 @@ sdhci_start_command(struct sdhci_slot *slot, struct mmc_command *cmd)
 	if (!SDHCI_GET_CARD_PRESENT(slot->bus, slot) ||
 	    slot->power == 0 ||
 	    slot->clock == 0) {
+		slot_printf(slot,
+			    "Cannot issue a command (power=%d clock=%d)",
+			    slot->power, slot->clock);
 		cmd->error = MMC_ERR_FAILED;
 		sdhci_req_done(slot);
 		return;
@@ -1103,10 +1602,20 @@ sdhci_start_command(struct sdhci_slot *slot, struct mmc_command *cmd)
 	/* Always wait for free CMD bus. */
 	mask = SDHCI_CMD_INHIBIT;
 	/* Wait for free DAT if we have data or busy signal. */
-	if (cmd->data || (cmd->flags & MMC_RSP_BUSY))
+	if (cmd->data != NULL || (cmd->flags & MMC_RSP_BUSY))
 		mask |= SDHCI_DAT_INHIBIT;
-	/* We shouldn't wait for DAT for stop commands. */
-	if (cmd == slot->req->stop)
+	/*
+	 * We shouldn't wait for DAT for stop commands or CMD19/CMD21.  Note
+	 * that these latter are also special in that SDHCI_CMD_DATA should
+	 * be set below but no actual data is ever read from the controller.
+	*/
+#ifdef MMCCAM
+	if (cmd == &slot->ccb->mmcio.stop ||
+#else
+	if (cmd == slot->req->stop ||
+#endif
+	    __predict_false(cmd->opcode == MMC_SEND_TUNING_BLOCK ||
+	    cmd->opcode == MMC_SEND_TUNING_BLOCK_HS200))
 		mask &= ~SDHCI_DAT_INHIBIT;
 	/*
 	 *  Wait for bus no more then 250 ms.  Typically there will be no wait
@@ -1145,7 +1654,7 @@ sdhci_start_command(struct sdhci_slot *slot, struct mmc_command *cmd)
 		flags |= SDHCI_CMD_CRC;
 	if (cmd->flags & MMC_RSP_OPCODE)
 		flags |= SDHCI_CMD_INDEX;
-	if (cmd->data)
+	if (cmd->data != NULL)
 		flags |= SDHCI_CMD_DATA;
 	if (cmd->opcode == MMC_STOP_TRANSMISSION)
 		flags |= SDHCI_CMD_TYPE_ABORT;
@@ -1164,6 +1673,8 @@ sdhci_start_command(struct sdhci_slot *slot, struct mmc_command *cmd)
 	WR4(slot, SDHCI_ARGUMENT, cmd->arg);
 	/* Set data transfer mode. */
 	sdhci_set_transfer_mode(slot, cmd->data);
+	if (__predict_false(sdhci_debug > 1))
+		slot_printf(slot, "Starting command!\n");
 	/* Start command. */
 	WR2(slot, SDHCI_COMMAND_FLAGS, (cmd->opcode << 8) | (flags & 0xff));
 	/* Start timeout callout. */
@@ -1178,15 +1689,23 @@ sdhci_finish_command(struct sdhci_slot *slot)
 	uint32_t val;
 	uint8_t extra;
 
+	if (__predict_false(sdhci_debug > 1))
+		slot_printf(slot, "%s: called, err %d flags %d\n",
+		    __func__, slot->curcmd->error, slot->curcmd->flags);
 	slot->cmd_done = 1;
 	/*
 	 * Interrupt aggregation: Restore command interrupt.
 	 * Main restore point for the case when command interrupt
 	 * happened first.
 	 */
-	WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask |= SDHCI_INT_RESPONSE);
+	if (__predict_true(slot->curcmd->opcode != MMC_SEND_TUNING_BLOCK &&
+	    slot->curcmd->opcode != MMC_SEND_TUNING_BLOCK_HS200))
+		WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask |=
+		    SDHCI_INT_RESPONSE);
 	/* In case of error - reset host and return. */
 	if (slot->curcmd->error) {
+		if (slot->curcmd->error == MMC_ERR_BADCRC)
+			slot->retune_req |= SDHCI_RETUNE_REQ_RESET;
 		sdhci_reset(slot, SDHCI_RESET_CMD);
 		sdhci_reset(slot, SDHCI_RESET_DATA);
 		sdhci_start(slot);
@@ -1211,6 +1730,11 @@ sdhci_finish_command(struct sdhci_slot *slot)
 		} else
 			slot->curcmd->resp[0] = RD4(slot, SDHCI_RESPONSE);
 	}
+	if (__predict_false(sdhci_debug > 1))
+		printf("Resp: %02x %02x %02x %02x\n",
+		    slot->curcmd->resp[0], slot->curcmd->resp[1],
+		    slot->curcmd->resp[2], slot->curcmd->resp[3]);
+
 	/* If data ready - finish. */
 	if (slot->data_done)
 		sdhci_start(slot);
@@ -1291,6 +1815,11 @@ sdhci_start_data(struct sdhci_slot *slot, struct mmc_data *data)
 	    (data->len < 512) ? data->len : 512));
 	/* Set block count. */
 	WR2(slot, SDHCI_BLOCK_COUNT, (data->len + 511) / 512);
+
+	if (__predict_false(sdhci_debug > 1))
+		slot_printf(slot, "Block size: %02x, count %lu\n",
+		    (unsigned int)SDHCI_MAKE_BLKSZ(DMA_BOUNDARY, (data->len < 512) ? data->len : 512),
+		    (unsigned long)(data->len + 511) / 512);
 }
 
 void
@@ -1322,6 +1851,8 @@ sdhci_finish_data(struct sdhci_slot *slot)
 	slot->data_done = 1;
 	/* If there was error - reset the host. */
 	if (slot->curcmd->error) {
+		if (slot->curcmd->error == MMC_ERR_BADCRC)
+			slot->retune_req |= SDHCI_RETUNE_REQ_RESET;
 		sdhci_reset(slot, SDHCI_RESET_CMD);
 		sdhci_reset(slot, SDHCI_RESET_DATA);
 		sdhci_start(slot);
@@ -1332,6 +1863,47 @@ sdhci_finish_data(struct sdhci_slot *slot)
 		sdhci_start(slot);
 }
 
+#ifdef MMCCAM
+static void
+sdhci_start(struct sdhci_slot *slot)
+{
+        union ccb *ccb;
+
+	ccb = slot->ccb;
+	if (ccb == NULL)
+		return;
+
+        struct ccb_mmcio *mmcio;
+	mmcio = &ccb->mmcio;
+
+	if (!(slot->flags & CMD_STARTED)) {
+		slot->flags |= CMD_STARTED;
+		sdhci_start_command(slot, &mmcio->cmd);
+		return;
+	}
+
+	/*
+	 * Old stack doesn't use this!
+	 * Enabling this code causes significant performance degradation
+	 * and IRQ storms on BBB, Wandboard behaves fine.
+	 * Not using this code does no harm...
+	if (!(slot->flags & STOP_STARTED) && mmcio->stop.opcode != 0) {
+		slot->flags |= STOP_STARTED;
+		sdhci_start_command(slot, &mmcio->stop);
+		return;
+	}
+	*/
+	if (__predict_false(sdhci_debug > 1))
+		slot_printf(slot, "result: %d\n", mmcio->cmd.error);
+	if (mmcio->cmd.error == 0 &&
+	    (slot->quirks & SDHCI_QUIRK_RESET_AFTER_REQUEST)) {
+		sdhci_reset(slot, SDHCI_RESET_CMD);
+		sdhci_reset(slot, SDHCI_RESET_DATA);
+	}
+
+	sdhci_req_done(slot);
+}
+#else
 static void
 sdhci_start(struct sdhci_slot *slot)
 {
@@ -1352,7 +1924,7 @@ sdhci_start(struct sdhci_slot *slot)
 		sdhci_start_command(slot, req->stop);
 		return;
 	}
-	if (sdhci_debug > 1)
+	if (__predict_false(sdhci_debug > 1))
 		slot_printf(slot, "result: %d\n", req->cmd->error);
 	if (!req->cmd->error &&
 	    ((slot->curcmd == req->stop &&
@@ -1364,6 +1936,7 @@ sdhci_start(struct sdhci_slot *slot)
 
 	sdhci_req_done(slot);
 }
+#endif
 
 int
 sdhci_generic_request(device_t brdev __unused, device_t reqdev,
@@ -1376,7 +1949,7 @@ sdhci_generic_request(device_t brdev __unused, device_t reqdev,
 		SDHCI_UNLOCK(slot);
 		return (EBUSY);
 	}
-	if (sdhci_debug > 1) {
+	if (__predict_false(sdhci_debug > 1)) {
 		slot_printf(slot,
 		    "CMD%u arg %#x flags %#x dlen %u dflags %#x\n",
 		    req->cmd->opcode, req->cmd->arg, req->cmd->flags,
@@ -1495,6 +2068,15 @@ sdhci_data_irq(struct sdhci_slot *slot, uint32_t intmask)
 		goto done;
 	}
 
+	/* Handle tuning completion interrupt. */
+	if (__predict_false((intmask & SDHCI_INT_DATA_AVAIL) &&
+	    (slot->curcmd->opcode == MMC_SEND_TUNING_BLOCK ||
+	    slot->curcmd->opcode == MMC_SEND_TUNING_BLOCK_HS200))) {
+		slot->req->flags |= MMC_TUNE_DONE;
+		sdhci_finish_command(slot);
+		sdhci_finish_data(slot);
+		return;
+	}
 	/* Handle PIO interrupt. */
 	if (intmask & (SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL)) {
 		if ((slot->opt & SDHCI_PLATFORM_TRANSFER) &&
@@ -1587,9 +2169,21 @@ sdhci_generic_intr(struct sdhci_slot *slot)
 		SDHCI_UNLOCK(slot);
 		return;
 	}
-	if (sdhci_debug > 2)
+	if (__predict_false(sdhci_debug > 2))
 		slot_printf(slot, "Interrupt %#x\n", intmask);
 
+	/* Handle tuning error interrupt. */
+	if (__predict_false(intmask & SDHCI_INT_TUNEERR)) {
+		slot_printf(slot, "Tuning error indicated\n");
+		slot->retune_req |= SDHCI_RETUNE_REQ_RESET;
+		if (slot->curcmd) {
+			slot->curcmd->error = MMC_ERR_BADCRC;
+			sdhci_finish_command(slot);
+		}
+	}
+	/* Handle re-tuning interrupt. */
+	if (__predict_false(intmask & SDHCI_INT_RETUNE))
+		slot->retune_req |= SDHCI_RETUNE_REQ_NEEDED;
 	/* Handle card presence interrupts. */
 	if (intmask & (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE)) {
 		present = (intmask & SDHCI_INT_CARD_INSERT) != 0;
@@ -1602,7 +2196,6 @@ sdhci_generic_intr(struct sdhci_slot *slot)
 		WR4(slot, SDHCI_INT_STATUS, intmask &
 		    (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE));
 		sdhci_handle_card_present_locked(slot, present);
-		intmask &= ~(SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE);
 	}
 	/* Handle command interrupts. */
 	if (intmask & SDHCI_INT_CMD_MASK) {
@@ -1621,16 +2214,14 @@ sdhci_generic_intr(struct sdhci_slot *slot)
 		WR4(slot, SDHCI_INT_STATUS, SDHCI_INT_ACMD12ERR);
 		sdhci_acmd_irq(slot);
 	}
-	intmask &= ~(SDHCI_INT_CMD_MASK | SDHCI_INT_DATA_MASK);
-	intmask &= ~SDHCI_INT_ACMD12ERR;
-	intmask &= ~SDHCI_INT_ERROR;
 	/* Handle bus power interrupt. */
 	if (intmask & SDHCI_INT_BUS_POWER) {
 		WR4(slot, SDHCI_INT_STATUS, SDHCI_INT_BUS_POWER);
-		slot_printf(slot,
-		    "Card is consuming too much power!\n");
-		intmask &= ~SDHCI_INT_BUS_POWER;
+		slot_printf(slot, "Card is consuming too much power!\n");
 	}
+	intmask &= ~(SDHCI_INT_ERROR | SDHCI_INT_TUNEERR | SDHCI_INT_RETUNE |
+	    SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE | SDHCI_INT_CMD_MASK |
+	    SDHCI_INT_DATA_MASK | SDHCI_INT_ACMD12ERR | SDHCI_INT_BUS_POWER);
 	/* The rest is unknown. */
 	if (intmask) {
 		WR4(slot, SDHCI_INT_STATUS, intmask);
@@ -1684,6 +2275,19 @@ sdhci_generic_read_ivar(device_t bus, device_t child, int which,
 	case MMCBR_IVAR_VDD:
 		*result = slot->host.ios.vdd;
 		break;
+	case MMCBR_IVAR_RETUNE_REQ:
+		if (slot->opt & SDHCI_TUNING_ENABLED) {
+			if (slot->retune_req & SDHCI_RETUNE_REQ_RESET) {
+				*result = retune_req_reset;
+				break;
+			}
+			if (slot->retune_req & SDHCI_RETUNE_REQ_NEEDED) {
+				*result = retune_req_normal;
+				break;
+			}
+		}
+		*result = retune_req_none;
+		break;
 	case MMCBR_IVAR_VCCQ:
 		*result = slot->host.ios.vccq;
 		break;
@@ -1694,6 +2298,16 @@ sdhci_generic_read_ivar(device_t bus, device_t child, int which,
 		*result = slot->host.ios.timing;
 		break;
 	case MMCBR_IVAR_MAX_DATA:
+		/*
+		 * Re-tuning modes 1 and 2 restrict the maximum data length
+		 * per read/write command to 4 MiB.
+		 */
+		if (slot->opt & SDHCI_TUNING_ENABLED &&
+		    (slot->retune_mode == SDHCI_RETUNE_MODE_1 ||
+		    slot->retune_mode == SDHCI_RETUNE_MODE_2)) {
+			*result = 4 * 1024 * 1024 / MMC_SECTOR_SIZE;
+			break;
+		}
 		*result = 65535;
 		break;
 	case MMCBR_IVAR_MAX_BUSY_TIMEOUT:
@@ -1714,6 +2328,8 @@ sdhci_generic_write_ivar(device_t bus, device_t child, int which,
 	uint32_t clock, max_clock;
 	int i;
 
+	if (sdhci_debug > 1)
+		slot_printf(slot, "%s: var=%d\n", __func__, which);
 	switch (which) {
 	default:
 		return (EINVAL);
@@ -1774,9 +2390,324 @@ sdhci_generic_write_ivar(device_t bus, device_t child, int which,
 	case MMCBR_IVAR_F_MIN:
 	case MMCBR_IVAR_F_MAX:
 	case MMCBR_IVAR_MAX_DATA:
+	case MMCBR_IVAR_RETUNE_REQ:
 		return (EINVAL);
 	}
 	return (0);
 }
 
+#ifdef MMCCAM
+void
+sdhci_cam_start_slot(struct sdhci_slot *slot)
+{
+        if ((slot->devq = cam_simq_alloc(1)) == NULL) {
+                goto fail;
+        }
+
+        mtx_init(&slot->sim_mtx, "sdhcisim", NULL, MTX_DEF);
+        slot->sim = cam_sim_alloc(sdhci_cam_action, sdhci_cam_poll,
+                                  "sdhci_slot", slot, device_get_unit(slot->bus),
+                                  &slot->sim_mtx, 1, 1, slot->devq);
+
+        if (slot->sim == NULL) {
+                cam_simq_free(slot->devq);
+                slot_printf(slot, "cannot allocate CAM SIM\n");
+                goto fail;
+        }
+
+        mtx_lock(&slot->sim_mtx);
+        if (xpt_bus_register(slot->sim, slot->bus, 0) != 0) {
+                slot_printf(slot,
+                              "cannot register SCSI pass-through bus\n");
+                cam_sim_free(slot->sim, FALSE);
+                cam_simq_free(slot->devq);
+                mtx_unlock(&slot->sim_mtx);
+                goto fail;
+        }
+
+        mtx_unlock(&slot->sim_mtx);
+        /* End CAM-specific init */
+	slot->card_present = 0;
+	sdhci_card_task(slot, 0);
+        return;
+
+fail:
+        if (slot->sim != NULL) {
+                mtx_lock(&slot->sim_mtx);
+                xpt_bus_deregister(cam_sim_path(slot->sim));
+                cam_sim_free(slot->sim, FALSE);
+                mtx_unlock(&slot->sim_mtx);
+        }
+
+        if (slot->devq != NULL)
+                cam_simq_free(slot->devq);
+}
+
+static void
+sdhci_cam_handle_mmcio(struct cam_sim *sim, union ccb *ccb)
+{
+	struct sdhci_slot *slot;
+
+	slot = cam_sim_softc(sim);
+
+	sdhci_cam_request(slot, ccb);
+}
+
+void
+sdhci_cam_action(struct cam_sim *sim, union ccb *ccb)
+{
+	struct sdhci_slot *slot;
+
+	slot = cam_sim_softc(sim);
+	if (slot == NULL) {
+		ccb->ccb_h.status = CAM_SEL_TIMEOUT;
+		xpt_done(ccb);
+		return;
+	}
+
+	mtx_assert(&slot->sim_mtx, MA_OWNED);
+
+	switch (ccb->ccb_h.func_code) {
+	case XPT_PATH_INQ:
+	{
+		struct ccb_pathinq *cpi;
+
+		cpi = &ccb->cpi;
+		cpi->version_num = 1;
+		cpi->hba_inquiry = 0;
+		cpi->target_sprt = 0;
+		cpi->hba_misc = PIM_NOBUSRESET | PIM_SEQSCAN;
+		cpi->hba_eng_cnt = 0;
+		cpi->max_target = 0;
+		cpi->max_lun = 0;
+		cpi->initiator_id = 1;
+		cpi->maxio = MAXPHYS;
+		strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
+		strncpy(cpi->hba_vid, "Deglitch Networks", HBA_IDLEN);
+		strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
+		cpi->unit_number = cam_sim_unit(sim);
+		cpi->bus_id = cam_sim_bus(sim);
+		cpi->base_transfer_speed = 100; /* XXX WTF? */
+		cpi->protocol = PROTO_MMCSD;
+		cpi->protocol_version = SCSI_REV_0;
+		cpi->transport = XPORT_MMCSD;
+		cpi->transport_version = 0;
+
+		cpi->ccb_h.status = CAM_REQ_CMP;
+		break;
+	}
+	case XPT_GET_TRAN_SETTINGS:
+	{
+		struct ccb_trans_settings *cts = &ccb->cts;
+
+		if (sdhci_debug > 1)
+			slot_printf(slot, "Got XPT_GET_TRAN_SETTINGS\n");
+
+		cts->protocol = PROTO_MMCSD;
+		cts->protocol_version = 1;
+		cts->transport = XPORT_MMCSD;
+		cts->transport_version = 1;
+		cts->xport_specific.valid = 0;
+		cts->proto_specific.mmc.host_ocr = slot->host.host_ocr;
+		cts->proto_specific.mmc.host_f_min = slot->host.f_min;
+		cts->proto_specific.mmc.host_f_max = slot->host.f_max;
+		cts->proto_specific.mmc.host_caps = slot->host.caps;
+		memcpy(&cts->proto_specific.mmc.ios, &slot->host.ios, sizeof(struct mmc_ios));
+		ccb->ccb_h.status = CAM_REQ_CMP;
+		break;
+	}
+	case XPT_SET_TRAN_SETTINGS:
+	{
+		if (sdhci_debug > 1)
+			slot_printf(slot, "Got XPT_SET_TRAN_SETTINGS\n");
+		sdhci_cam_settran_settings(slot, ccb);
+		ccb->ccb_h.status = CAM_REQ_CMP;
+		break;
+	}
+	case XPT_RESET_BUS:
+		if (sdhci_debug > 1)
+			slot_printf(slot, "Got XPT_RESET_BUS, ACK it...\n");
+		ccb->ccb_h.status = CAM_REQ_CMP;
+		break;
+	case XPT_MMC_IO:
+		/*
+		 * Here is the HW-dependent part of
+		 * sending the command to the underlying h/w
+		 * At some point in the future an interrupt comes.
+		 * Then the request will be marked as completed.
+		 */
+		if (__predict_false(sdhci_debug > 1))
+			slot_printf(slot, "Got XPT_MMC_IO\n");
+		ccb->ccb_h.status = CAM_REQ_INPROG;
+
+		sdhci_cam_handle_mmcio(sim, ccb);
+		return;
+		/* NOTREACHED */
+		break;
+	default:
+		ccb->ccb_h.status = CAM_REQ_INVALID;
+		break;
+	}
+	xpt_done(ccb);
+	return;
+}
+
+void
+sdhci_cam_poll(struct cam_sim *sim)
+{
+	return;
+}
+
+int sdhci_cam_get_possible_host_clock(struct sdhci_slot *slot, int proposed_clock) {
+	int max_clock, clock, i;
+
+	if (proposed_clock == 0)
+		return 0;
+	max_clock = slot->max_clk;
+	clock = max_clock;
+
+	if (slot->version < SDHCI_SPEC_300) {
+		for (i = 0; i < SDHCI_200_MAX_DIVIDER;
+		     i <<= 1) {
+			if (clock <= proposed_clock)
+				break;
+			clock >>= 1;
+		}
+	} else {
+		for (i = 0; i < SDHCI_300_MAX_DIVIDER;
+		     i += 2) {
+			if (clock <= proposed_clock)
+				break;
+			clock = max_clock / (i + 2);
+		}
+	}
+	return clock;
+}
+
+int
+sdhci_cam_settran_settings(struct sdhci_slot *slot, union ccb *ccb)
+{
+	struct mmc_ios *ios;
+	struct mmc_ios *new_ios;
+	struct ccb_trans_settings_mmc *cts;
+
+	ios = &slot->host.ios;
+
+	cts = &ccb->cts.proto_specific.mmc;
+	new_ios = &cts->ios;
+
+	/* Update only requested fields */
+	if (cts->ios_valid & MMC_CLK) {
+		ios->clock = sdhci_cam_get_possible_host_clock(slot, new_ios->clock);
+		slot_printf(slot, "Clock => %d\n", ios->clock);
+	}
+	if (cts->ios_valid & MMC_VDD) {
+		ios->vdd = new_ios->vdd;
+		slot_printf(slot, "VDD => %d\n", ios->vdd);
+	}
+	if (cts->ios_valid & MMC_CS) {
+		ios->chip_select = new_ios->chip_select;
+		slot_printf(slot, "CS => %d\n", ios->chip_select);
+	}
+	if (cts->ios_valid & MMC_BW) {
+		ios->bus_width = new_ios->bus_width;
+		slot_printf(slot, "Bus width => %d\n", ios->bus_width);
+	}
+	if (cts->ios_valid & MMC_PM) {
+		ios->power_mode = new_ios->power_mode;
+		slot_printf(slot, "Power mode => %d\n", ios->power_mode);
+	}
+	if (cts->ios_valid & MMC_BT) {
+		ios->timing = new_ios->timing;
+		slot_printf(slot, "Timing => %d\n", ios->timing);
+	}
+	if (cts->ios_valid & MMC_BM) {
+		ios->bus_mode = new_ios->bus_mode;
+		slot_printf(slot, "Bus mode => %d\n", ios->bus_mode);
+	}
+
+        /* XXX Provide a way to call a chip-specific IOS update, required for TI */
+	return (sdhci_cam_update_ios(slot));
+}
+
+int
+sdhci_cam_update_ios(struct sdhci_slot *slot)
+{
+	struct mmc_ios *ios = &slot->host.ios;
+
+	slot_printf(slot, "%s: power_mode=%d, clk=%d, bus_width=%d, timing=%d\n",
+		    __func__, ios->power_mode, ios->clock, ios->bus_width, ios->timing);
+	SDHCI_LOCK(slot);
+	/* Do full reset on bus power down to clear from any state. */
+	if (ios->power_mode == power_off) {
+		WR4(slot, SDHCI_SIGNAL_ENABLE, 0);
+		sdhci_init(slot);
+	}
+	/* Configure the bus. */
+	sdhci_set_clock(slot, ios->clock);
+	sdhci_set_power(slot, (ios->power_mode == power_off) ? 0 : ios->vdd);
+	if (ios->bus_width == bus_width_8) {
+		slot->hostctrl |= SDHCI_CTRL_8BITBUS;
+		slot->hostctrl &= ~SDHCI_CTRL_4BITBUS;
+	} else if (ios->bus_width == bus_width_4) {
+		slot->hostctrl &= ~SDHCI_CTRL_8BITBUS;
+		slot->hostctrl |= SDHCI_CTRL_4BITBUS;
+	} else if (ios->bus_width == bus_width_1) {
+		slot->hostctrl &= ~SDHCI_CTRL_8BITBUS;
+		slot->hostctrl &= ~SDHCI_CTRL_4BITBUS;
+	} else {
+		panic("Invalid bus width: %d", ios->bus_width);
+	}
+	if (ios->timing == bus_timing_hs &&
+	    !(slot->quirks & SDHCI_QUIRK_DONT_SET_HISPD_BIT))
+		slot->hostctrl |= SDHCI_CTRL_HISPD;
+	else
+		slot->hostctrl &= ~SDHCI_CTRL_HISPD;
+	WR1(slot, SDHCI_HOST_CONTROL, slot->hostctrl);
+	/* Some controllers like reset after bus changes. */
+	if(slot->quirks & SDHCI_QUIRK_RESET_ON_IOS)
+		sdhci_reset(slot, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
+
+	SDHCI_UNLOCK(slot);
+	return (0);
+}
+
+int
+sdhci_cam_request(struct sdhci_slot *slot, union ccb *ccb)
+{
+	struct ccb_mmcio *mmcio;
+
+	mmcio = &ccb->mmcio;
+
+	SDHCI_LOCK(slot);
+/*	if (slot->req != NULL) {
+		SDHCI_UNLOCK(slot);
+		return (EBUSY);
+	}
+*/
+	if (__predict_false(sdhci_debug > 1)) {
+		slot_printf(slot, "CMD%u arg %#x flags %#x dlen %u dflags %#x\n",
+			    mmcio->cmd.opcode, mmcio->cmd.arg, mmcio->cmd.flags,
+			    mmcio->cmd.data != NULL ? (unsigned int) mmcio->cmd.data->len : 0,
+			    mmcio->cmd.data != NULL ? mmcio->cmd.data->flags: 0);
+	}
+	if (mmcio->cmd.data != NULL) {
+		if (mmcio->cmd.data->len == 0 || mmcio->cmd.data->flags == 0)
+			panic("data->len = %d, data->flags = %d -- something is b0rked",
+			      (int)mmcio->cmd.data->len, mmcio->cmd.data->flags);
+	}
+	slot->ccb = ccb;
+	slot->flags = 0;
+	sdhci_start(slot);
+	SDHCI_UNLOCK(slot);
+	if (dumping) {
+		while (slot->ccb != NULL) {
+			sdhci_generic_intr(slot);
+			DELAY(10);
+		}
+	}
+	return (0);
+}
+#endif /* MMCCAM */
+
 MODULE_VERSION(sdhci, 1);