Update to FreeBSD head 2019-09-24

Git mirror commit 6b0307a0a5184339393f555d5d424190d8a8277a.

1 files changed, 481 insertions, 43 deletions

diff --git a/freebsd/sys/dev/nvme/nvme.h b/freebsd/sys/dev/nvme/nvme.h
index 845ba75b..0d4a18b7 100644
--- a/freebsd/sys/dev/nvme/nvme.h
+++ b/freebsd/sys/dev/nvme/nvme.h
@@ -40,6 +40,7 @@
 
 #define	NVME_PASSTHROUGH_CMD		_IOWR('n', 0, struct nvme_pt_command)
 #define	NVME_RESET_CONTROLLER		_IO('n', 1)
+#define	NVME_GET_NSID			_IOR('n', 2, struct nvme_get_nsid)
 
 #define	NVME_IO_TEST			_IOWR('n', 100, struct nvme_io_test)
 #define	NVME_BIO_TEST			_IOWR('n', 101, struct nvme_io_test)
@@ -69,15 +70,39 @@
 #define NVME_CAP_LO_REG_AMS_MASK			(0x3)
 #define NVME_CAP_LO_REG_TO_SHIFT			(24)
 #define NVME_CAP_LO_REG_TO_MASK				(0xFF)
+#define NVME_CAP_LO_MQES(x) \
+	(((x) >> NVME_CAP_LO_REG_MQES_SHIFT) & NVME_CAP_LO_REG_MQES_MASK)
+#define NVME_CAP_LO_CQR(x) \
+	(((x) >> NVME_CAP_LO_REG_CQR_SHIFT) & NVME_CAP_LO_REG_CQR_MASK)
+#define NVME_CAP_LO_AMS(x) \
+	(((x) >> NVME_CAP_LO_REG_AMS_SHIFT) & NVME_CAP_LO_REG_AMS_MASK)
+#define NVME_CAP_LO_TO(x) \
+	(((x) >> NVME_CAP_LO_REG_TO_SHIFT) & NVME_CAP_LO_REG_TO_MASK)
 
 #define NVME_CAP_HI_REG_DSTRD_SHIFT			(0)
 #define NVME_CAP_HI_REG_DSTRD_MASK			(0xF)
+#define NVME_CAP_HI_REG_NSSRS_SHIFT			(4)
+#define NVME_CAP_HI_REG_NSSRS_MASK			(0x1)
 #define NVME_CAP_HI_REG_CSS_NVM_SHIFT			(5)
 #define NVME_CAP_HI_REG_CSS_NVM_MASK			(0x1)
+#define NVME_CAP_HI_REG_BPS_SHIFT			(13)
+#define NVME_CAP_HI_REG_BPS_MASK			(0x1)
 #define NVME_CAP_HI_REG_MPSMIN_SHIFT			(16)
 #define NVME_CAP_HI_REG_MPSMIN_MASK			(0xF)
 #define NVME_CAP_HI_REG_MPSMAX_SHIFT			(20)
 #define NVME_CAP_HI_REG_MPSMAX_MASK			(0xF)
+#define NVME_CAP_HI_REG_PMRS_SHIFT			(24)
+#define NVME_CAP_HI_REG_PMRS_MASK			(0x1)
+#define NVME_CAP_HI_REG_CMBS_SHIFT			(25)
+#define NVME_CAP_HI_REG_CMBS_MASK			(0x1)
+#define NVME_CAP_HI_DSTRD(x) \
+	(((x) >> NVME_CAP_HI_REG_DSTRD_SHIFT) & NVME_CAP_HI_REG_DSTRD_MASK)
+#define NVME_CAP_HI_CSS_NVM(x) \
+	(((x) >> NVME_CAP_HI_REG_CSS_NVM_SHIFT) & NVME_CAP_HI_REG_CSS_NVM_MASK)
+#define NVME_CAP_HI_MPSMIN(x) \
+	(((x) >> NVME_CAP_HI_REG_MPSMIN_SHIFT) & NVME_CAP_HI_REG_MPSMIN_MASK)
+#define NVME_CAP_HI_MPSMAX(x) \
+	(((x) >> NVME_CAP_HI_REG_MPSMAX_SHIFT) & NVME_CAP_HI_REG_MPSMAX_MASK)
 
 #define NVME_CC_REG_EN_SHIFT				(0)
 #define NVME_CC_REG_EN_MASK				(0x1)
@@ -100,6 +125,10 @@
 #define NVME_CSTS_REG_CFS_MASK				(0x1)
 #define NVME_CSTS_REG_SHST_SHIFT			(2)
 #define NVME_CSTS_REG_SHST_MASK				(0x3)
+#define NVME_CSTS_REG_NVSRO_SHIFT			(4)
+#define NVME_CSTS_REG_NVSRO_MASK			(0x1)
+#define NVME_CSTS_REG_PP_SHIFT				(5)
+#define NVME_CSTS_REG_PP_MASK				(0x1)
 
 #define NVME_CSTS_GET_SHST(csts)			(((csts) >> NVME_CSTS_REG_SHST_SHIFT) & NVME_CSTS_REG_SHST_MASK)
 
@@ -119,6 +148,8 @@
 #define NVME_STATUS_SC_MASK				(0xFF)
 #define NVME_STATUS_SCT_SHIFT				(9)
 #define NVME_STATUS_SCT_MASK				(0x7)
+#define NVME_STATUS_CRD_SHIFT				(12)
+#define NVME_STATUS_CRD_MASK				(0x3)
 #define NVME_STATUS_M_SHIFT				(14)
 #define NVME_STATUS_M_MASK				(0x1)
 #define NVME_STATUS_DNR_SHIFT				(15)
@@ -159,6 +190,9 @@
 /* SR-IOV Virtual Function */
 #define NVME_CTRLR_DATA_MIC_SRIOVVF_SHIFT		(2)
 #define NVME_CTRLR_DATA_MIC_SRIOVVF_MASK		(0x1)
+/* Asymmetric Namespace Access Reporting */
+#define NVME_CTRLR_DATA_MIC_ANAR_SHIFT			(3)
+#define NVME_CTRLR_DATA_MIC_ANAR_MASK			(0x1)
 
 /** OACS - optional admin command support */
 /* supports security send/receive commands */
@@ -188,6 +222,9 @@
 /* supports Doorbell Buffer Config */
 #define NVME_CTRLR_DATA_OACS_DBBUFFER_SHIFT		(8)
 #define NVME_CTRLR_DATA_OACS_DBBUFFER_MASK		(0x1)
+/* supports Get LBA Status */
+#define NVME_CTRLR_DATA_OACS_GETLBA_SHIFT		(9)
+#define NVME_CTRLR_DATA_OACS_GETLBA_MASK		(0x1)
 
 /** firmware updates */
 /* first slot is read-only */
@@ -196,6 +233,9 @@
 /* number of firmware slots */
 #define NVME_CTRLR_DATA_FRMW_NUM_SLOTS_SHIFT		(1)
 #define NVME_CTRLR_DATA_FRMW_NUM_SLOTS_MASK		(0x7)
+/* firmware activation without reset */
+#define NVME_CTRLR_DATA_FRMW_ACT_WO_RESET_SHIFT		(4)
+#define NVME_CTRLR_DATA_FRMW_ACT_WO_RESET_MASK		(0x1)
 
 /** log page attributes */
 /* per namespace smart/health log page */
@@ -212,6 +252,26 @@
 #define NVME_CTRLR_DATA_APSTA_APST_SUPP_SHIFT		(0)
 #define NVME_CTRLR_DATA_APSTA_APST_SUPP_MASK		(0x1)
 
+/** Sanitize Capabilities */
+/* Crypto Erase Support  */
+#define NVME_CTRLR_DATA_SANICAP_CES_SHIFT		(0)
+#define NVME_CTRLR_DATA_SANICAP_CES_MASK		(0x1)
+/* Block Erase Support */
+#define NVME_CTRLR_DATA_SANICAP_BES_SHIFT		(1)
+#define NVME_CTRLR_DATA_SANICAP_BES_MASK		(0x1)
+/* Overwrite Support */
+#define NVME_CTRLR_DATA_SANICAP_OWS_SHIFT		(2)
+#define NVME_CTRLR_DATA_SANICAP_OWS_MASK		(0x1)
+/* No-Deallocate Inhibited  */
+#define NVME_CTRLR_DATA_SANICAP_NDI_SHIFT		(29)
+#define NVME_CTRLR_DATA_SANICAP_NDI_MASK		(0x1)
+/* No-Deallocate Modifies Media After Sanitize */
+#define NVME_CTRLR_DATA_SANICAP_NODMMAS_SHIFT		(30)
+#define NVME_CTRLR_DATA_SANICAP_NODMMAS_MASK		(0x3)
+#define NVME_CTRLR_DATA_SANICAP_NODMMAS_UNDEF		(0)
+#define NVME_CTRLR_DATA_SANICAP_NODMMAS_NO		(1)
+#define NVME_CTRLR_DATA_SANICAP_NODMMAS_YES		(2)
+
 /** submission queue entry size */
 #define NVME_CTRLR_DATA_SQES_MIN_SHIFT			(0)
 #define NVME_CTRLR_DATA_SQES_MIN_MASK			(0xF)
@@ -239,6 +299,8 @@
 #define NVME_CTRLR_DATA_ONCS_RESERV_MASK		(0x1)
 #define NVME_CTRLR_DATA_ONCS_TIMESTAMP_SHIFT		(6)
 #define NVME_CTRLR_DATA_ONCS_TIMESTAMP_MASK		(0x1)
+#define NVME_CTRLR_DATA_ONCS_VERIFY_SHIFT		(7)
+#define NVME_CTRLR_DATA_ONCS_VERIFY_MASK		(0x1)
 
 /** Fused Operation Support */
 #define NVME_CTRLR_DATA_FUSES_CNW_SHIFT		(0)
@@ -253,8 +315,15 @@
 #define NVME_CTRLR_DATA_FNA_CRYPTO_ERASE_MASK		(0x1)
 
 /** volatile write cache */
+/* volatile write cache present */
 #define NVME_CTRLR_DATA_VWC_PRESENT_SHIFT		(0)
 #define NVME_CTRLR_DATA_VWC_PRESENT_MASK		(0x1)
+/* flush all namespaces supported */
+#define NVME_CTRLR_DATA_VWC_ALL_SHIFT			(1)
+#define NVME_CTRLR_DATA_VWC_ALL_MASK			(0x3)
+#define NVME_CTRLR_DATA_VWC_ALL_UNKNOWN			(0)
+#define NVME_CTRLR_DATA_VWC_ALL_NO			(2)
+#define NVME_CTRLR_DATA_VWC_ALL_YES			(3)
 
 /** namespace features */
 /* thin provisioning */
@@ -269,6 +338,9 @@
 /* NGUID and EUI64 fields are not reusable */
 #define NVME_NS_DATA_NSFEAT_NO_ID_REUSE_SHIFT		(3)
 #define NVME_NS_DATA_NSFEAT_NO_ID_REUSE_MASK		(0x1)
+/* NPWG, NPWA, NPDG, NPDA, and NOWS are valid */
+#define NVME_NS_DATA_NSFEAT_NPVALID_SHIFT		(4)
+#define NVME_NS_DATA_NSFEAT_NPVALID_MASK		(0x1)
 
 /** formatted lba size */
 #define NVME_NS_DATA_FLBAS_FORMAT_SHIFT			(0)
@@ -349,6 +421,20 @@
 #define NVME_NS_DATA_FPI_SUPP_SHIFT		(7)
 #define NVME_NS_DATA_FPI_SUPP_MASK		(0x1)
 
+/** Deallocate Logical Block Features */
+/* deallocated logical block read behavior */
+#define NVME_NS_DATA_DLFEAT_READ_SHIFT		(0)
+#define NVME_NS_DATA_DLFEAT_READ_MASK		(0x07)
+#define NVME_NS_DATA_DLFEAT_READ_NR		(0x00)
+#define NVME_NS_DATA_DLFEAT_READ_00		(0x01)
+#define NVME_NS_DATA_DLFEAT_READ_FF		(0x02)
+/* supports the Deallocate bit in the Write Zeroes */
+#define NVME_NS_DATA_DLFEAT_DWZ_SHIFT		(3)
+#define NVME_NS_DATA_DLFEAT_DWZ_MASK		(0x01)
+/* Guard field for deallocated logical blocks is set to the CRC  */
+#define NVME_NS_DATA_DLFEAT_GCRC_SHIFT		(4)
+#define NVME_NS_DATA_DLFEAT_GCRC_MASK		(0x01)
+
 /** lba format support */
 /* metadata size */
 #define NVME_NS_DATA_LBAF_MS_SHIFT			(0)
@@ -373,6 +459,35 @@ enum nvme_critical_warning_state {
 #define NVME_FIRMWARE_PAGE_AFI_SLOT_SHIFT		(0)
 #define NVME_FIRMWARE_PAGE_AFI_SLOT_MASK		(0x7)
 
+/* Commands Supported and Effects */
+#define	NVME_CE_PAGE_CSUP_SHIFT				(0)
+#define	NVME_CE_PAGE_CSUP_MASK				(0x1)
+#define	NVME_CE_PAGE_LBCC_SHIFT				(1)
+#define	NVME_CE_PAGE_LBCC_MASK				(0x1)
+#define	NVME_CE_PAGE_NCC_SHIFT				(2)
+#define	NVME_CE_PAGE_NCC_MASK				(0x1)
+#define	NVME_CE_PAGE_NIC_SHIFT				(3)
+#define	NVME_CE_PAGE_NIC_MASK				(0x1)
+#define	NVME_CE_PAGE_CCC_SHIFT				(4)
+#define	NVME_CE_PAGE_CCC_MASK				(0x1)
+#define	NVME_CE_PAGE_CSE_SHIFT				(16)
+#define	NVME_CE_PAGE_CSE_MASK				(0x7)
+#define	NVME_CE_PAGE_UUID_SHIFT				(19)
+#define	NVME_CE_PAGE_UUID_MASK				(0x1)
+
+/* Sanitize Status */
+#define	NVME_SS_PAGE_SSTAT_STATUS_SHIFT			(0)
+#define	NVME_SS_PAGE_SSTAT_STATUS_MASK			(0x7)
+#define	NVME_SS_PAGE_SSTAT_STATUS_NEVER			(0)
+#define	NVME_SS_PAGE_SSTAT_STATUS_COMPLETED		(1)
+#define	NVME_SS_PAGE_SSTAT_STATUS_INPROG		(2)
+#define	NVME_SS_PAGE_SSTAT_STATUS_FAILED		(3)
+#define	NVME_SS_PAGE_SSTAT_STATUS_COMPLETEDWD		(4)
+#define	NVME_SS_PAGE_SSTAT_PASSES_SHIFT			(3)
+#define	NVME_SS_PAGE_SSTAT_PASSES_MASK			(0x1f)
+#define	NVME_SS_PAGE_SSTAT_GDE_SHIFT			(8)
+#define	NVME_SS_PAGE_SSTAT_GDE_MASK			(0x1)
+
 /* CC register SHN field values */
 enum shn_value {
 	NVME_SHN_NORMAL		= 0x1,
@@ -388,34 +503,37 @@ enum shst_value {
 
 struct nvme_registers
 {
-	/** controller capabilities */
-	uint32_t		cap_lo;
-	uint32_t		cap_hi;
-
-	uint32_t		vs;	/* version */
-	uint32_t		intms;	/* interrupt mask set */
-	uint32_t		intmc;	/* interrupt mask clear */
-
-	/** controller configuration */
-	uint32_t		cc;
-
-	uint32_t		reserved1;
-
-	/** controller status */
-	uint32_t		csts;
-
-	uint32_t		reserved2;
-
-	/** admin queue attributes */
-	uint32_t		aqa;
-
-	uint64_t		asq;	/* admin submission queue base addr */
-	uint64_t		acq;	/* admin completion queue base addr */
-	uint32_t		reserved3[0x3f2];
-
+	uint32_t	cap_lo; /* controller capabilities */
+	uint32_t	cap_hi;
+	uint32_t	vs;	/* version */
+	uint32_t	intms;	/* interrupt mask set */
+	uint32_t	intmc;	/* interrupt mask clear */
+	uint32_t	cc;	/* controller configuration */
+	uint32_t	reserved1;
+	uint32_t	csts;	/* controller status */
+	uint32_t	nssr;	/* NVM Subsystem Reset */
+	uint32_t	aqa;	/* admin queue attributes */
+	uint64_t	asq;	/* admin submission queue base addr */
+	uint64_t	acq;	/* admin completion queue base addr */
+	uint32_t	cmbloc;	/* Controller Memory Buffer Location */
+	uint32_t	cmbsz;	/* Controller Memory Buffer Size */
+	uint32_t	bpinfo;	/* Boot Partition Information */
+	uint32_t	bprsel;	/* Boot Partition Read Select */
+	uint64_t	bpmbl;	/* Boot Partition Memory Buffer Location */
+	uint64_t	cmbmsc;	/* Controller Memory Buffer Memory Space Control */
+	uint32_t	cmbsts;	/* Controller Memory Buffer Status */
+	uint8_t		reserved3[3492]; /* 5Ch - DFFh */
+	uint32_t	pmrcap;	/* Persistent Memory Capabilities */
+	uint32_t	pmrctl;	/* Persistent Memory Region Control */
+	uint32_t	pmrsts;	/* Persistent Memory Region Status */
+	uint32_t	pmrebs;	/* Persistent Memory Region Elasticity Buffer Size */
+	uint32_t	pmrswtp; /* Persistent Memory Region Sustained Write Throughput */
+	uint32_t	pmrmsc_lo; /* Persistent Memory Region Controller Memory Space Control */
+	uint32_t	pmrmsc_hi;
+	uint8_t		reserved4[484]; /* E1Ch - FFFh */
 	struct {
-	    uint32_t		sq_tdbl; /* submission queue tail doorbell */
-	    uint32_t		cq_hdbl; /* completion queue head doorbell */
+	    uint32_t	sq_tdbl; /* submission queue tail doorbell */
+	    uint32_t	cq_hdbl; /* completion queue head doorbell */
 	} doorbell[1] __packed;
 } __packed;
 
@@ -490,6 +608,7 @@ enum nvme_status_code_type {
 	NVME_SCT_GENERIC		= 0x0,
 	NVME_SCT_COMMAND_SPECIFIC	= 0x1,
 	NVME_SCT_MEDIA_ERROR		= 0x2,
+	NVME_SCT_PATH_RELATED		= 0x3,
 	/* 0x3-0x6 - reserved */
 	NVME_SCT_VENDOR_SPECIFIC	= 0x7,
 };
@@ -528,6 +647,9 @@ enum nvme_generic_command_status_code {
 	NVME_SC_SANITIZE_IN_PROGRESS		= 0x1d,
 	NVME_SC_SGL_DATA_BLOCK_GRAN_INVALID	= 0x1e,
 	NVME_SC_NOT_SUPPORTED_IN_CMB		= 0x1f,
+	NVME_SC_NAMESPACE_IS_WRITE_PROTECTED	= 0x20,
+	NVME_SC_COMMAND_INTERRUPTED		= 0x21,
+	NVME_SC_TRANSIENT_TRANSPORT_ERROR	= 0x22,
 
 	NVME_SC_LBA_OUT_OF_RANGE		= 0x80,
 	NVME_SC_CAPACITY_EXCEEDED		= 0x81,
@@ -573,6 +695,9 @@ enum nvme_command_specific_status_code {
 	NVME_SC_INVALID_SEC_CTRLR_STATE		= 0x20,
 	NVME_SC_INVALID_NUM_OF_CTRLR_RESRC	= 0x21,
 	NVME_SC_INVALID_RESOURCE_ID		= 0x22,
+	NVME_SC_SANITIZE_PROHIBITED_WPMRE	= 0x23,
+	NVME_SC_ANA_GROUP_ID_INVALID		= 0x24,
+	NVME_SC_ANA_ATTACH_FAILED		= 0x25,
 
 	NVME_SC_CONFLICTING_ATTRIBUTES		= 0x80,
 	NVME_SC_INVALID_PROTECTION_INFO		= 0x81,
@@ -591,6 +716,17 @@ enum nvme_media_error_status_code {
 	NVME_SC_DEALLOCATED_OR_UNWRITTEN	= 0x87,
 };
 
+/* path related status codes */
+enum nvme_path_related_status_code {
+	NVME_SC_INTERNAL_PATH_ERROR		= 0x00,
+	NVME_SC_ASYMMETRIC_ACCESS_PERSISTENT_LOSS = 0x01,
+	NVME_SC_ASYMMETRIC_ACCESS_INACCESSIBLE	= 0x02,
+	NVME_SC_ASYMMETRIC_ACCESS_TRANSITION	= 0x03,
+	NVME_SC_CONTROLLER_PATHING_ERROR	= 0x60,
+	NVME_SC_HOST_PATHING_ERROR		= 0x70,
+	NVME_SC_COMMAND_ABOTHED_BY_HOST		= 0x71,
+};
+
 /* admin opcodes */
 enum nvme_admin_opcode {
 	NVME_OPC_DELETE_IO_SQ			= 0x00,
@@ -610,20 +746,27 @@ enum nvme_admin_opcode {
 	/* 0x0e-0x0f - reserved */
 	NVME_OPC_FIRMWARE_ACTIVATE		= 0x10,
 	NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD	= 0x11,
+	/* 0x12-0x13 - reserved */
 	NVME_OPC_DEVICE_SELF_TEST		= 0x14,
 	NVME_OPC_NAMESPACE_ATTACHMENT		= 0x15,
+	/* 0x16-0x17 - reserved */
 	NVME_OPC_KEEP_ALIVE			= 0x18,
 	NVME_OPC_DIRECTIVE_SEND			= 0x19,
 	NVME_OPC_DIRECTIVE_RECEIVE		= 0x1a,
+	/* 0x1b - reserved */
 	NVME_OPC_VIRTUALIZATION_MANAGEMENT	= 0x1c,
 	NVME_OPC_NVME_MI_SEND			= 0x1d,
 	NVME_OPC_NVME_MI_RECEIVE		= 0x1e,
+	/* 0x1f-0x7b - reserved */
 	NVME_OPC_DOORBELL_BUFFER_CONFIG		= 0x7c,
 
 	NVME_OPC_FORMAT_NVM			= 0x80,
 	NVME_OPC_SECURITY_SEND			= 0x81,
 	NVME_OPC_SECURITY_RECEIVE		= 0x82,
+	/* 0x83 - reserved */
 	NVME_OPC_SANITIZE			= 0x84,
+	/* 0x85 - reserved */
+	NVME_OPC_GET_LBA_STATUS			= 0x86,
 };
 
 /* nvme nvm opcodes */
@@ -634,11 +777,11 @@ enum nvme_nvm_opcode {
 	/* 0x03 - reserved */
 	NVME_OPC_WRITE_UNCORRECTABLE		= 0x04,
 	NVME_OPC_COMPARE			= 0x05,
-	/* 0x06 - reserved */
+	/* 0x06-0x07 - reserved */
 	NVME_OPC_WRITE_ZEROES			= 0x08,
-	/* 0x07 - reserved */
 	NVME_OPC_DATASET_MANAGEMENT		= 0x09,
-	/* 0x0a-0x0c - reserved */
+	/* 0x0a-0x0b - reserved */
+	NVME_OPC_VERIFY				= 0x0c,
 	NVME_OPC_RESERVATION_REGISTER		= 0x0d,
 	NVME_OPC_RESERVATION_REPORT		= 0x0e,
 	/* 0x0f-0x10 - reserved */
@@ -666,10 +809,21 @@ enum nvme_feature {
 	NVME_FEAT_KEEP_ALIVE_TIMER		= 0x0F,
 	NVME_FEAT_HOST_CONTROLLED_THERMAL_MGMT	= 0x10,
 	NVME_FEAT_NON_OP_POWER_STATE_CONFIG	= 0x11,
-	/* 0x12-0x77 - reserved */
+	NVME_FEAT_READ_RECOVERY_LEVEL_CONFIG	= 0x12,
+	NVME_FEAT_PREDICTABLE_LATENCY_MODE_CONFIG = 0x13,
+	NVME_FEAT_PREDICTABLE_LATENCY_MODE_WINDOW = 0x14,
+	NVME_FEAT_LBA_STATUS_INFORMATION_ATTRIBUTES = 0x15,
+	NVME_FEAT_HOST_BEHAVIOR_SUPPORT		= 0x16,
+	NVME_FEAT_SANITIZE_CONFIG		= 0x17,
+	NVME_FEAT_ENDURANCE_GROUP_EVENT_CONFIGURATION = 0x18,
+	/* 0x19-0x77 - reserved */
 	/* 0x78-0x7f - NVMe Management Interface */
 	NVME_FEAT_SOFTWARE_PROGRESS_MARKER	= 0x80,
-	/* 0x81-0xBF - command set specific (reserved) */
+	NVME_FEAT_HOST_IDENTIFIER		= 0x81,
+	NVME_FEAT_RESERVATION_NOTIFICATION_MASK	= 0x82,
+	NVME_FEAT_RESERVATION_PERSISTENCE	= 0x83,
+	NVME_FEAT_NAMESPACE_WRITE_PROTECTION_CONFIG = 0x84,
+	/* 0x85-0xBF - command set specific (reserved) */
 	/* 0xC0-0xFF - vendor specific */
 };
 
@@ -763,12 +917,27 @@ struct nvme_controller_data {
 	/** Controller Attributes */
 	uint32_t		ctratt;	/* bitfield really */
 
-	uint8_t			reserved1[12];
+	/** Read Recovery Levels Supported */
+	uint16_t		rrls;
+
+	uint8_t			reserved1[9];
+
+	/** Controller Type */
+	uint8_t			cntrltype;
 
 	/** FRU Globally Unique Identifier */
 	uint8_t			fguid[16];
 
-	uint8_t			reserved2[128];
+	/** Command Retry Delay Time 1 */
+	uint16_t		crdt1;
+
+	/** Command Retry Delay Time 2 */
+	uint16_t		crdt2;
+
+	/** Command Retry Delay Time 3 */
+	uint16_t		crdt3;
+
+	uint8_t			reserved2[122];
 
 	/* bytes 256-511: admin command set attributes */
 
@@ -848,7 +1017,34 @@ struct nvme_controller_data {
 	/** Sanitize Capabilities */
 	uint32_t		sanicap; /* Really a bitfield */
 
-	uint8_t			reserved3[180];
+	/** Host Memory Buffer Minimum Descriptor Entry Size */
+	uint32_t		hmminds;
+
+	/** Host Memory Maximum Descriptors Entries */
+	uint16_t		hmmaxd;
+
+	/** NVM Set Identifier Maximum */
+	uint16_t		nsetidmax;
+
+	/** Endurance Group Identifier Maximum */
+	uint16_t		endgidmax;
+
+	/** ANA Transition Time */
+	uint8_t			anatt;
+
+	/** Asymmetric Namespace Access Capabilities */
+	uint8_t			anacap;
+
+	/** ANA Group Identifier Maximum */
+	uint32_t		anagrpmax;
+
+	/** Number of ANA Group Identifiers */
+	uint32_t		nanagrpid;
+
+	/** Persistent Event Log Size */
+	uint32_t		pels;
+
+	uint8_t			reserved3[156];
 	/* bytes 512-703: nvm command set attributes */
 
 	/** submission queue entry size */
@@ -883,7 +1079,9 @@ struct nvme_controller_data {
 
 	/** NVM Vendor Specific Command Configuration */
 	uint8_t			nvscc;
-	uint8_t			reserved5;
+
+	/** Namespace Write Protection Capabilities */
+	uint8_t			nwpc;
 
 	/** Atomic Compare & Write Unit */
 	uint16_t		acwu;
@@ -892,8 +1090,11 @@ struct nvme_controller_data {
 	/** SGL Support */
 	uint32_t		sgls;
 
+	/** Maximum Number of Allowed Namespaces */
+	uint32_t		mnan;
+
 	/* bytes 540-767: Reserved */
-	uint8_t			reserved7[228];
+	uint8_t			reserved7[224];
 
 	/** NVM Subsystem NVMe Qualified Name */
 	uint8_t			subnqn[256];
@@ -978,8 +1179,38 @@ struct nvme_namespace_data {
 	/** NVM Capacity */
 	uint8_t			nvmcap[16];
 
-	/* bytes 64-103: Reserved */
-	uint8_t			reserved5[40];
+	/** Namespace Preferred Write Granularity  */
+	uint16_t		npwg;
+
+	/** Namespace Preferred Write Alignment */
+	uint16_t		npwa;
+
+	/** Namespace Preferred Deallocate Granularity */
+	uint16_t		npdg;
+
+	/** Namespace Preferred Deallocate Alignment */
+	uint16_t		npda;
+
+	/** Namespace Optimal Write Size */
+	uint16_t		nows;
+
+	/* bytes 74-91: Reserved */
+	uint8_t			reserved5[18];
+
+	/** ANA Group Identifier */
+	uint32_t		anagrpid;
+
+	/* bytes 96-98: Reserved */
+	uint8_t			reserved6[3];
+
+	/** Namespace Attributes */
+	uint8_t			nsattr;
+
+	/** NVM Set Identifier */
+	uint16_t		nvmsetid;
+
+	/** Endurance Group Identifier */
+	uint16_t		endgid;
 
 	/** Namespace Globally Unique Identifier */
 	uint8_t			nguid[16];
@@ -990,7 +1221,7 @@ struct nvme_namespace_data {
 	/** lba format support */
 	uint32_t		lbaf[16];
 
-	uint8_t			reserved6[192];
+	uint8_t			reserved7[192];
 
 	uint8_t			vendor_specific[3712];
 } __packed __aligned(4);
@@ -1005,9 +1236,21 @@ enum nvme_log_page {
 	NVME_LOG_FIRMWARE_SLOT		= 0x03,
 	NVME_LOG_CHANGED_NAMESPACE	= 0x04,
 	NVME_LOG_COMMAND_EFFECT		= 0x05,
+	NVME_LOG_DEVICE_SELF_TEST	= 0x06,
+	NVME_LOG_TELEMETRY_HOST_INITIATED = 0x07,
+	NVME_LOG_TELEMETRY_CONTROLLER_INITIATED = 0x08,
+	NVME_LOG_ENDURANCE_GROUP_INFORMATION = 0x09,
+	NVME_LOG_PREDICTABLE_LATENCY_PER_NVM_SET = 0x0a,
+	NVME_LOG_PREDICTABLE_LATENCY_EVENT_AGGREGATE = 0x0b,
+	NVME_LOG_ASYMMETRIC_NAMESPAVE_ACCESS = 0x0c,
+	NVME_LOG_PERSISTENT_EVENT_LOG	= 0x0d,
+	NVME_LOG_LBA_STATUS_INFORMATION	= 0x0e,
+	NVME_LOG_ENDURANCE_GROUP_EVENT_AGGREGATE = 0x0f,
 	/* 0x06-0x7F - reserved */
 	/* 0x80-0xBF - I/O command set specific */
 	NVME_LOG_RES_NOTIFICATION	= 0x80,
+	NVME_LOG_SANITIZE_STATUS	= 0x81,
+	/* 0x82-0xBF - reserved */
 	/* 0xC0-0xFF - vendor specific */
 
 	/*
@@ -1036,7 +1279,11 @@ struct nvme_error_information_entry {
 	uint64_t		lba;
 	uint32_t		nsid;
 	uint8_t			vendor_specific;
-	uint8_t			reserved[35];
+	uint8_t			trtype;
+	uint16_t		reserved30;
+	uint64_t		csi;
+	uint16_t		ttsi;
+	uint8_t			reserved[22];
 } __packed __aligned(4);
 
 _Static_assert(sizeof(struct nvme_error_information_entry) == 64, "bad size for nvme_error_information_entry");
@@ -1072,8 +1319,16 @@ struct nvme_health_information_page {
 	uint32_t		warning_temp_time;
 	uint32_t		error_temp_time;
 	uint16_t		temp_sensor[8];
-
-	uint8_t			reserved2[296];
+	/* Thermal Management Temperature 1 Transition Count */
+	uint32_t		tmt1tc;
+	/* Thermal Management Temperature 2 Transition Count */
+	uint32_t		tmt2tc;
+	/* Total Time For Thermal Management Temperature 1 */
+	uint32_t		ttftmt1;
+	/* Total Time For Thermal Management Temperature 2 */
+	uint32_t		ttftmt2;
+
+	uint8_t			reserved2[280];
 } __packed __aligned(4);
 
 _Static_assert(sizeof(struct nvme_health_information_page) == 512, "bad size for nvme_health_information_page");
@@ -1094,6 +1349,43 @@ struct nvme_ns_list {
 
 _Static_assert(sizeof(struct nvme_ns_list) == 4096, "bad size for nvme_ns_list");
 
+struct nvme_command_effects_page {
+	uint32_t		acs[256];
+	uint32_t		iocs[256];
+	uint8_t			reserved[2048];
+} __packed __aligned(4);
+
+_Static_assert(sizeof(struct nvme_command_effects_page) == 4096,
+    "bad size for nvme_command_effects_page");
+
+struct nvme_res_notification_page {
+	uint64_t		log_page_count;
+	uint8_t			log_page_type;
+	uint8_t			available_log_pages;
+	uint8_t			reserved2;
+	uint32_t		nsid;
+	uint8_t			reserved[48];
+} __packed __aligned(4);
+
+_Static_assert(sizeof(struct nvme_res_notification_page) == 64,
+    "bad size for nvme_res_notification_page");
+
+struct nvme_sanitize_status_page {
+	uint16_t		sprog;
+	uint16_t		sstat;
+	uint32_t		scdw10;
+	uint32_t		etfo;
+	uint32_t		etfbe;
+	uint32_t		etfce;
+	uint32_t		etfownd;
+	uint32_t		etfbewnd;
+	uint32_t		etfcewnd;
+	uint8_t			reserved[480];
+} __packed __aligned(4);
+
+_Static_assert(sizeof(struct nvme_sanitize_status_page) == 512,
+    "bad size for nvme_sanitize_status_page");
+
 struct intel_log_temp_stats
 {
 	uint64_t	current;
@@ -1109,6 +1401,56 @@ struct intel_log_temp_stats
 
 _Static_assert(sizeof(struct intel_log_temp_stats) == 13 * 8, "bad size for intel_log_temp_stats");
 
+struct nvme_resv_reg_ctrlr
+{
+	uint16_t		ctrlr_id;	/* Controller ID */
+	uint8_t			rcsts;		/* Reservation Status */
+	uint8_t			reserved3[5];
+	uint64_t		hostid;		/* Host Identifier */
+	uint64_t		rkey;		/* Reservation Key */
+} __packed __aligned(4);
+
+_Static_assert(sizeof(struct nvme_resv_reg_ctrlr) == 24, "bad size for nvme_resv_reg_ctrlr");
+
+struct nvme_resv_reg_ctrlr_ext
+{
+	uint16_t		ctrlr_id;	/* Controller ID */
+	uint8_t			rcsts;		/* Reservation Status */
+	uint8_t			reserved3[5];
+	uint64_t		rkey;		/* Reservation Key */
+	uint64_t		hostid[2];	/* Host Identifier */
+	uint8_t			reserved32[32];
+} __packed __aligned(4);
+
+_Static_assert(sizeof(struct nvme_resv_reg_ctrlr_ext) == 64, "bad size for nvme_resv_reg_ctrlr_ext");
+
+struct nvme_resv_status
+{
+	uint32_t		gen;		/* Generation */
+	uint8_t			rtype;		/* Reservation Type */
+	uint8_t			regctl[2];	/* Number of Registered Controllers */
+	uint8_t			reserved7[2];
+	uint8_t			ptpls;		/* Persist Through Power Loss State */
+	uint8_t			reserved10[14];
+	struct nvme_resv_reg_ctrlr	ctrlr[0];
+} __packed __aligned(4);
+
+_Static_assert(sizeof(struct nvme_resv_status) == 24, "bad size for nvme_resv_status");
+
+struct nvme_resv_status_ext
+{
+	uint32_t		gen;		/* Generation */
+	uint8_t			rtype;		/* Reservation Type */
+	uint8_t			regctl[2];	/* Number of Registered Controllers */
+	uint8_t			reserved7[2];
+	uint8_t			ptpls;		/* Persist Through Power Loss State */
+	uint8_t			reserved10[14];
+	uint8_t			reserved24[40];
+	struct nvme_resv_reg_ctrlr_ext	ctrlr[0];
+} __packed __aligned(4);
+
+_Static_assert(sizeof(struct nvme_resv_status_ext) == 64, "bad size for nvme_resv_status_ext");
+
 #define NVME_TEST_MAX_THREADS	128
 
 struct nvme_io_test {
@@ -1184,6 +1526,11 @@ struct nvme_pt_command {
 	struct mtx *		driver_lock;
 };
 
+struct nvme_get_nsid {
+	char		cdev[SPECNAMELEN + 1];
+	uint32_t	nsid;
+};
+
 #define nvme_completion_is_error(cpl)					\
 	(NVME_STATUS_GET_SC((cpl)->status) != 0 || NVME_STATUS_GET_SCT((cpl)->status) != 0)
 
@@ -1192,6 +1539,7 @@ void	nvme_strvis(uint8_t *dst, const uint8_t *src, int dstlen, int srclen);
 #ifdef _KERNEL
 
 struct bio;
+struct thread;
 
 struct nvme_namespace;
 struct nvme_controller;
@@ -1281,6 +1629,8 @@ uint32_t	nvme_ns_get_stripesize(struct nvme_namespace *ns);
 
 int	nvme_ns_bio_process(struct nvme_namespace *ns, struct bio *bp,
 			    nvme_cb_fn_t cb_fn);
+int	nvme_ns_ioctl_process(struct nvme_namespace *ns, u_long cmd,
+    caddr_t arg, int flag, struct thread *td);
 
 /*
  * Command building helper functions -- shared with CAM
@@ -1372,6 +1722,10 @@ void	nvme_controller_data_swapbytes(struct nvme_controller_data *s)
 	s->rtd3e = le32toh(s->rtd3e);
 	s->oaes = le32toh(s->oaes);
 	s->ctratt = le32toh(s->ctratt);
+	s->rrls = le16toh(s->rrls);
+	s->crdt1 = le16toh(s->crdt1);
+	s->crdt2 = le16toh(s->crdt2);
+	s->crdt3 = le16toh(s->crdt3);
 	s->oacs = le16toh(s->oacs);
 	s->wctemp = le16toh(s->wctemp);
 	s->cctemp = le16toh(s->cctemp);
@@ -1385,6 +1739,13 @@ void	nvme_controller_data_swapbytes(struct nvme_controller_data *s)
 	s->mntmt = le16toh(s->mntmt);
 	s->mxtmt = le16toh(s->mxtmt);
 	s->sanicap = le32toh(s->sanicap);
+	s->hmminds = le32toh(s->hmminds);
+	s->hmmaxd = le16toh(s->hmmaxd);
+	s->nsetidmax = le16toh(s->nsetidmax);
+	s->endgidmax = le16toh(s->endgidmax);
+	s->anagrpmax = le32toh(s->anagrpmax);
+	s->nanagrpid = le32toh(s->nanagrpid);
+	s->pels = le32toh(s->pels);
 	s->maxcmd = le16toh(s->maxcmd);
 	s->nn = le32toh(s->nn);
 	s->oncs = le16toh(s->oncs);
@@ -1393,6 +1754,7 @@ void	nvme_controller_data_swapbytes(struct nvme_controller_data *s)
 	s->awupf = le16toh(s->awupf);
 	s->acwu = le16toh(s->acwu);
 	s->sgls = le32toh(s->sgls);
+	s->mnan = le32toh(s->mnan);
 	for (i = 0; i < 32; i++)
 		nvme_power_state_swapbytes(&s->power_state[i]);
 }
@@ -1412,6 +1774,14 @@ void	nvme_namespace_data_swapbytes(struct nvme_namespace_data *s)
 	s->nabo = le16toh(s->nabo);
 	s->nabspf = le16toh(s->nabspf);
 	s->noiob = le16toh(s->noiob);
+	s->npwg = le16toh(s->npwg);
+	s->npwa = le16toh(s->npwa);
+	s->npdg = le16toh(s->npdg);
+	s->npda = le16toh(s->npda);
+	s->nows = le16toh(s->nows);
+	s->anagrpid = le32toh(s->anagrpid);
+	s->nvmsetid = le16toh(s->nvmsetid);
+	s->endgid = le16toh(s->endgid);
 	for (i = 0; i < 16; i++)
 		s->lbaf[i] = le32toh(s->lbaf[i]);
 }
@@ -1427,6 +1797,8 @@ void	nvme_error_information_entry_swapbytes(struct nvme_error_information_entry
 	s->error_location = le16toh(s->error_location);
 	s->lba = le64toh(s->lba);
 	s->nsid = le32toh(s->nsid);
+	s->csi = le64toh(s->csi);
+	s->ttsi = le16toh(s->ttsi);
 }
 
 static inline
@@ -1467,6 +1839,10 @@ void	nvme_health_information_page_swapbytes(struct nvme_health_information_page
 	s->error_temp_time = le32toh(s->error_temp_time);
 	for (i = 0; i < 8; i++)
 		s->temp_sensor[i] = le16toh(s->temp_sensor[i]);
+	s->tmt1tc = le32toh(s->tmt1tc);
+	s->tmt2tc = le32toh(s->tmt2tc);
+	s->ttftmt1 = le32toh(s->ttftmt1);
+	s->ttftmt2 = le32toh(s->ttftmt2);
 }
 
 
@@ -1489,6 +1865,38 @@ void	nvme_ns_list_swapbytes(struct nvme_ns_list *s)
 }
 
 static inline
+void	nvme_command_effects_page_swapbytes(struct nvme_command_effects_page *s)
+{
+	int i;
+
+	for (i = 0; i < 256; i++)
+		s->acs[i] = le32toh(s->acs[i]);
+	for (i = 0; i < 256; i++)
+		s->iocs[i] = le32toh(s->iocs[i]);
+}
+
+static inline
+void	nvme_res_notification_page_swapbytes(struct nvme_res_notification_page *s)
+{
+	s->log_page_count = le64toh(s->log_page_count);
+	s->nsid = le32toh(s->nsid);
+}
+
+static inline
+void	nvme_sanitize_status_page_swapbytes(struct nvme_sanitize_status_page *s)
+{
+	s->sprog = le16toh(s->sprog);
+	s->sstat = le16toh(s->sstat);
+	s->scdw10 = le32toh(s->scdw10);
+	s->etfo = le32toh(s->etfo);
+	s->etfbe = le32toh(s->etfbe);
+	s->etfce = le32toh(s->etfce);
+	s->etfownd = le32toh(s->etfownd);
+	s->etfbewnd = le32toh(s->etfbewnd);
+	s->etfcewnd = le32toh(s->etfcewnd);
+}
+
+static inline
 void	intel_log_temp_stats_swapbytes(struct intel_log_temp_stats *s)
 {
 
@@ -1503,4 +1911,34 @@ void	intel_log_temp_stats_swapbytes(struct intel_log_temp_stats *s)
 	s->est_offset = le64toh(s->est_offset);
 }
 
+static inline
+void	nvme_resv_status_swapbytes(struct nvme_resv_status *s, size_t size)
+{
+	u_int i, n;
+
+	s->gen = le32toh(s->gen);
+	n = (s->regctl[1] << 8) | s->regctl[0];
+	n = MIN(n, (size - sizeof(s)) / sizeof(s->ctrlr[0]));
+	for (i = 0; i < n; i++) {
+		s->ctrlr[i].ctrlr_id = le16toh(s->ctrlr[i].ctrlr_id);
+		s->ctrlr[i].hostid = le64toh(s->ctrlr[i].hostid);
+		s->ctrlr[i].rkey = le64toh(s->ctrlr[i].rkey);
+	}
+}
+
+static inline
+void	nvme_resv_status_ext_swapbytes(struct nvme_resv_status_ext *s, size_t size)
+{
+	u_int i, n;
+
+	s->gen = le32toh(s->gen);
+	n = (s->regctl[1] << 8) | s->regctl[0];
+	n = MIN(n, (size - sizeof(s)) / sizeof(s->ctrlr[0]));
+	for (i = 0; i < n; i++) {
+		s->ctrlr[i].ctrlr_id = le16toh(s->ctrlr[i].ctrlr_id);
+		s->ctrlr[i].rkey = le64toh(s->ctrlr[i].rkey);
+		nvme_le128toh((void *)s->ctrlr[i].hostid);
+	}
+}
+
 #endif /* __NVME_H__ */