NVME(4): Add support for SGL

Update #3821.

4 files changed, 118 insertions, 0 deletions

diff --git a/freebsd/sys/dev/nvme/nvme.h b/freebsd/sys/dev/nvme/nvme.h
index 27259dc2..0a772fa4 100644
--- a/freebsd/sys/dev/nvme/nvme.h
+++ b/freebsd/sys/dev/nvme/nvme.h
@@ -37,6 +37,9 @@
 
 #include <sys/param.h>
 #include <sys/endian.h>
+#ifdef __rtems__
+#include <sys/_iovec.h>
+#endif /* __rtems__ */
 
 #define	NVME_PASSTHROUGH_CMD		_IOWR('n', 0, struct nvme_pt_command)
 #define	NVME_RESET_CONTROLLER		_IO('n', 1)
@@ -1607,11 +1610,23 @@ int	nvme_ns_cmd_write(struct nvme_namespace *ns, void *payload,
 			  void *cb_arg);
 int	nvme_ns_cmd_write_bio(struct nvme_namespace *ns, struct bio *bp,
 			      nvme_cb_fn_t cb_fn, void *cb_arg);
+#ifdef __rtems__
+int	nvme_ns_cmd_write_iov(struct nvme_namespace *ns,
+			      const struct iovec *iov, uint64_t lba,
+			      uint32_t lba_count, nvme_cb_fn_t cb_fn,
+			      void *cb_arg);
+#endif /* __rtems__ */
 int	nvme_ns_cmd_read(struct nvme_namespace *ns, void *payload,
 			 uint64_t lba, uint32_t lba_count, nvme_cb_fn_t cb_fn,
 			 void *cb_arg);
 int	nvme_ns_cmd_read_bio(struct nvme_namespace *ns, struct bio *bp,
 			      nvme_cb_fn_t cb_fn, void *cb_arg);
+#ifdef __rtems__
+int	nvme_ns_cmd_read_iov(struct nvme_namespace *ns,
+			     const struct iovec *iov, uint64_t lba,
+			     uint32_t lba_count, nvme_cb_fn_t cb_fn,
+			     void *cb_arg);
+#endif /* __rtems__ */
 int	nvme_ns_cmd_deallocate(struct nvme_namespace *ns, void *payload,
 			       uint8_t num_ranges, nvme_cb_fn_t cb_fn,
 			       void *cb_arg);
diff --git a/freebsd/sys/dev/nvme/nvme_ns_cmd.c b/freebsd/sys/dev/nvme/nvme_ns_cmd.c
index 70260869..00a9a7aa 100644
--- a/freebsd/sys/dev/nvme/nvme_ns_cmd.c
+++ b/freebsd/sys/dev/nvme/nvme_ns_cmd.c
@@ -74,6 +74,22 @@ nvme_ns_cmd_read_bio(struct nvme_namespace *ns, struct bio *bp,
 
 	return (0);
 }
+#else /* __rtems__ */
+int
+nvme_ns_cmd_read_iov(struct nvme_namespace *ns, const struct iovec *iov,
+    uint64_t lba, uint32_t lba_count, nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+
+	req = nvme_allocate_request_iov(iov,
+	    lba_count * nvme_ns_get_sector_size(ns), cb_fn, cb_arg);
+	if (req == NULL)
+		return (ENOMEM);
+
+	nvme_ns_read_cmd(&req->cmd, ns->id, lba, lba_count);
+	nvme_ctrlr_submit_io_request(ns->ctrlr, req);
+	return (0);
+}
 #endif /* __rtems__ */
 
 int
@@ -116,6 +132,22 @@ nvme_ns_cmd_write_bio(struct nvme_namespace *ns, struct bio *bp,
 
 	return (0);
 }
+#else /* __rtems__ */
+int
+nvme_ns_cmd_write_iov(struct nvme_namespace *ns, const struct iovec *iov,
+    uint64_t lba, uint32_t lba_count, nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+
+	req = nvme_allocate_request_iov(iov,
+	    lba_count * nvme_ns_get_sector_size(ns), cb_fn, cb_arg);
+	if (req == NULL)
+		return (ENOMEM);
+
+	nvme_ns_write_cmd(&req->cmd, ns->id, lba, lba_count);
+	nvme_ctrlr_submit_io_request(ns->ctrlr, req);
+	return (0);
+}
 #endif /* __rtems__ */
 
 int
diff --git a/freebsd/sys/dev/nvme/nvme_private.h b/freebsd/sys/dev/nvme/nvme_private.h
index 8346d5ea..ade1833c 100644
--- a/freebsd/sys/dev/nvme/nvme_private.h
+++ b/freebsd/sys/dev/nvme/nvme_private.h
@@ -130,6 +130,9 @@ extern devclass_t nvme_devclass;
 #define NVME_REQUEST_UIO	3
 #define NVME_REQUEST_BIO	4
 #define NVME_REQUEST_CCB        5
+#ifdef __rtems__
+#define NVME_REQUEST_IOV	10
+#endif /* __rtems__ */
 
 struct nvme_request {
 
@@ -138,6 +141,9 @@ struct nvme_request {
 	union {
 		void			*payload;
 		struct bio		*bio;
+#ifdef __rtems__
+		const struct iovec	*iov;
+#endif /* __rtems__ */
 	} u;
 	uint32_t			type;
 	uint32_t			payload_size;
@@ -556,6 +562,23 @@ nvme_allocate_request_ccb(union ccb *ccb, nvme_cb_fn_t cb_fn, void *cb_arg)
 
 	return (req);
 }
+#ifdef __rtems__
+static __inline struct nvme_request *
+nvme_allocate_request_iov(const struct iovec *iov, uint32_t payload_size,
+    nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+
+	req = _nvme_allocate_request(cb_fn, cb_arg);
+	if (req != NULL) {
+		req->type = NVME_REQUEST_IOV;
+		req->u.iov = iov;
+		req->payload_size = payload_size;
+	}
+
+	return (req);
+}
+#endif /* __rtems__ */
 
 #define nvme_free_request(req)	uma_zfree(nvme_request_zone, req)
 
diff --git a/freebsd/sys/dev/nvme/nvme_qpair.c b/freebsd/sys/dev/nvme/nvme_qpair.c
index a9d515d9..a866a959 100644
--- a/freebsd/sys/dev/nvme/nvme_qpair.c
+++ b/freebsd/sys/dev/nvme/nvme_qpair.c
@@ -1044,6 +1044,50 @@ nvme_payload_map(void *arg, bus_dma_segment_t *seg, int nseg, int error)
 #endif /* __rtems__ */
 	nvme_qpair_submit_tracker(tr->qpair, tr);
 }
+#ifdef __rtems__
+static void
+nvme_qpair_submit_request_iov(struct nvme_qpair *qpair,
+    struct nvme_request *req, struct nvme_tracker *tr)
+{
+	const struct iovec *iov;
+	size_t n;
+	size_t desc_count;
+	struct nvme_sgl_desc first;
+	struct nvme_sgl_desc *desc;
+
+	/* Enable SGL for this command */
+	req->cmd.fuse |= 0x40;
+
+	desc = (struct nvme_sgl_desc *)tr->prp;
+	desc_count = 0;
+	n = req->payload_size;
+	iov = req->u.iov;
+
+	while (n > 0) {
+		BSD_ASSERT(desc_count < (NVME_MAX_PRP_LIST_ENTRIES *
+		    sizeof(*tr->prp) / sizeof(*desc)));
+		desc->address = htole64((uintptr_t)iov->iov_base);
+		desc->length = htole32(iov->iov_len);
+		desc->reserved12[0] = 0;
+		desc->reserved12[1] = 0;
+		desc->reserved12[2] = 0;
+		desc->sgl_ident = NVME_SGL_IDENT_TYPE_DATA_BLOCK <<
+		    NVME_SGL_IDENT_TYPE_SHIFT;
+		BSD_ASSERT(n >= iov->iov_len);
+		n -= iov->iov_len;
+		++iov;
+		++desc;
+		++desc_count;
+	}
+
+	first.address = htole64((uint64_t)tr->prp_bus_addr);
+	first.length = htole32(desc_count * sizeof(*desc));
+	first.sgl_ident = NVME_SGL_IDENT_TYPE_LAST_SEG_DESC <<
+	    NVME_SGL_IDENT_TYPE_SHIFT;
+	memcpy(&req->cmd.prp1, &first, sizeof(first));
+	nvme_qpair_submit_tracker(tr->qpair, tr);
+}
+#endif /* __rtems__ */
 
 static void
 _nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req)
@@ -1152,6 +1196,10 @@ _nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req)
 			nvme_printf(qpair->ctrlr,
 			    "bus_dmamap_load_ccb returned 0x%x!\n", err);
 		break;
+#else /* __rtems__ */
+	case NVME_REQUEST_IOV:
+		nvme_qpair_submit_request_iov(tr->qpair, req, tr);
+		break;
 #endif /* __rtems__ */
 	default:
 		panic("unknown nvme request type 0x%x\n", req->type);