From ab4be9ec77c51939d94ff241b9746be38290a146 Mon Sep 17 00:00:00 2001 From: Sebastian Huber Date: Wed, 18 Sep 2019 13:39:58 +0200 Subject: NVME(4): Add support for SGL Update #3821. --- freebsd/sys/dev/nvme/nvme.h | 15 ++++++++++++ freebsd/sys/dev/nvme/nvme_ns_cmd.c | 32 +++++++++++++++++++++++++ freebsd/sys/dev/nvme/nvme_private.h | 23 ++++++++++++++++++ freebsd/sys/dev/nvme/nvme_qpair.c | 48 +++++++++++++++++++++++++++++++++++++ 4 files changed, 118 insertions(+) diff --git a/freebsd/sys/dev/nvme/nvme.h b/freebsd/sys/dev/nvme/nvme.h index 27259dc2..0a772fa4 100644 --- a/freebsd/sys/dev/nvme/nvme.h +++ b/freebsd/sys/dev/nvme/nvme.h @@ -37,6 +37,9 @@ #include #include +#ifdef __rtems__ +#include +#endif /* __rtems__ */ #define NVME_PASSTHROUGH_CMD _IOWR('n', 0, struct nvme_pt_command) #define NVME_RESET_CONTROLLER _IO('n', 1) @@ -1607,11 +1610,23 @@ int nvme_ns_cmd_write(struct nvme_namespace *ns, void *payload, void *cb_arg); int nvme_ns_cmd_write_bio(struct nvme_namespace *ns, struct bio *bp, nvme_cb_fn_t cb_fn, void *cb_arg); +#ifdef __rtems__ +int nvme_ns_cmd_write_iov(struct nvme_namespace *ns, + const struct iovec *iov, uint64_t lba, + uint32_t lba_count, nvme_cb_fn_t cb_fn, + void *cb_arg); +#endif /* __rtems__ */ int nvme_ns_cmd_read(struct nvme_namespace *ns, void *payload, uint64_t lba, uint32_t lba_count, nvme_cb_fn_t cb_fn, void *cb_arg); int nvme_ns_cmd_read_bio(struct nvme_namespace *ns, struct bio *bp, nvme_cb_fn_t cb_fn, void *cb_arg); +#ifdef __rtems__ +int nvme_ns_cmd_read_iov(struct nvme_namespace *ns, + const struct iovec *iov, uint64_t lba, + uint32_t lba_count, nvme_cb_fn_t cb_fn, + void *cb_arg); +#endif /* __rtems__ */ int nvme_ns_cmd_deallocate(struct nvme_namespace *ns, void *payload, uint8_t num_ranges, nvme_cb_fn_t cb_fn, void *cb_arg); diff --git a/freebsd/sys/dev/nvme/nvme_ns_cmd.c b/freebsd/sys/dev/nvme/nvme_ns_cmd.c index 70260869..00a9a7aa 100644 --- a/freebsd/sys/dev/nvme/nvme_ns_cmd.c +++ b/freebsd/sys/dev/nvme/nvme_ns_cmd.c @@ -74,6 +74,22 @@ nvme_ns_cmd_read_bio(struct nvme_namespace *ns, struct bio *bp, return (0); } +#else /* __rtems__ */ +int +nvme_ns_cmd_read_iov(struct nvme_namespace *ns, const struct iovec *iov, + uint64_t lba, uint32_t lba_count, nvme_cb_fn_t cb_fn, void *cb_arg) +{ + struct nvme_request *req; + + req = nvme_allocate_request_iov(iov, + lba_count * nvme_ns_get_sector_size(ns), cb_fn, cb_arg); + if (req == NULL) + return (ENOMEM); + + nvme_ns_read_cmd(&req->cmd, ns->id, lba, lba_count); + nvme_ctrlr_submit_io_request(ns->ctrlr, req); + return (0); +} #endif /* __rtems__ */ int @@ -116,6 +132,22 @@ nvme_ns_cmd_write_bio(struct nvme_namespace *ns, struct bio *bp, return (0); } +#else /* __rtems__ */ +int +nvme_ns_cmd_write_iov(struct nvme_namespace *ns, const struct iovec *iov, + uint64_t lba, uint32_t lba_count, nvme_cb_fn_t cb_fn, void *cb_arg) +{ + struct nvme_request *req; + + req = nvme_allocate_request_iov(iov, + lba_count * nvme_ns_get_sector_size(ns), cb_fn, cb_arg); + if (req == NULL) + return (ENOMEM); + + nvme_ns_write_cmd(&req->cmd, ns->id, lba, lba_count); + nvme_ctrlr_submit_io_request(ns->ctrlr, req); + return (0); +} #endif /* __rtems__ */ int diff --git a/freebsd/sys/dev/nvme/nvme_private.h b/freebsd/sys/dev/nvme/nvme_private.h index 8346d5ea..ade1833c 100644 --- a/freebsd/sys/dev/nvme/nvme_private.h +++ b/freebsd/sys/dev/nvme/nvme_private.h @@ -130,6 +130,9 @@ extern devclass_t nvme_devclass; #define NVME_REQUEST_UIO 3 #define NVME_REQUEST_BIO 4 #define NVME_REQUEST_CCB 5 +#ifdef __rtems__ +#define NVME_REQUEST_IOV 10 +#endif /* __rtems__ */ struct nvme_request { @@ -138,6 +141,9 @@ struct nvme_request { union { void *payload; struct bio *bio; +#ifdef __rtems__ + const struct iovec *iov; +#endif /* __rtems__ */ } u; uint32_t type; uint32_t payload_size; @@ -556,6 +562,23 @@ nvme_allocate_request_ccb(union ccb *ccb, nvme_cb_fn_t cb_fn, void *cb_arg) return (req); } +#ifdef __rtems__ +static __inline struct nvme_request * +nvme_allocate_request_iov(const struct iovec *iov, uint32_t payload_size, + nvme_cb_fn_t cb_fn, void *cb_arg) +{ + struct nvme_request *req; + + req = _nvme_allocate_request(cb_fn, cb_arg); + if (req != NULL) { + req->type = NVME_REQUEST_IOV; + req->u.iov = iov; + req->payload_size = payload_size; + } + + return (req); +} +#endif /* __rtems__ */ #define nvme_free_request(req) uma_zfree(nvme_request_zone, req) diff --git a/freebsd/sys/dev/nvme/nvme_qpair.c b/freebsd/sys/dev/nvme/nvme_qpair.c index a9d515d9..a866a959 100644 --- a/freebsd/sys/dev/nvme/nvme_qpair.c +++ b/freebsd/sys/dev/nvme/nvme_qpair.c @@ -1044,6 +1044,50 @@ nvme_payload_map(void *arg, bus_dma_segment_t *seg, int nseg, int error) #endif /* __rtems__ */ nvme_qpair_submit_tracker(tr->qpair, tr); } +#ifdef __rtems__ +static void +nvme_qpair_submit_request_iov(struct nvme_qpair *qpair, + struct nvme_request *req, struct nvme_tracker *tr) +{ + const struct iovec *iov; + size_t n; + size_t desc_count; + struct nvme_sgl_desc first; + struct nvme_sgl_desc *desc; + + /* Enable SGL for this command */ + req->cmd.fuse |= 0x40; + + desc = (struct nvme_sgl_desc *)tr->prp; + desc_count = 0; + n = req->payload_size; + iov = req->u.iov; + + while (n > 0) { + BSD_ASSERT(desc_count < (NVME_MAX_PRP_LIST_ENTRIES * + sizeof(*tr->prp) / sizeof(*desc))); + desc->address = htole64((uintptr_t)iov->iov_base); + desc->length = htole32(iov->iov_len); + desc->reserved12[0] = 0; + desc->reserved12[1] = 0; + desc->reserved12[2] = 0; + desc->sgl_ident = NVME_SGL_IDENT_TYPE_DATA_BLOCK << + NVME_SGL_IDENT_TYPE_SHIFT; + BSD_ASSERT(n >= iov->iov_len); + n -= iov->iov_len; + ++iov; + ++desc; + ++desc_count; + } + + first.address = htole64((uint64_t)tr->prp_bus_addr); + first.length = htole32(desc_count * sizeof(*desc)); + first.sgl_ident = NVME_SGL_IDENT_TYPE_LAST_SEG_DESC << + NVME_SGL_IDENT_TYPE_SHIFT; + memcpy(&req->cmd.prp1, &first, sizeof(first)); + nvme_qpair_submit_tracker(tr->qpair, tr); +} +#endif /* __rtems__ */ static void _nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req) @@ -1152,6 +1196,10 @@ _nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req) nvme_printf(qpair->ctrlr, "bus_dmamap_load_ccb returned 0x%x!\n", err); break; +#else /* __rtems__ */ + case NVME_REQUEST_IOV: + nvme_qpair_submit_request_iov(tr->qpair, req, tr); + break; #endif /* __rtems__ */ default: panic("unknown nvme request type 0x%x\n", req->type); -- cgit v1.2.3