From 6878519aea7a32eb1fa563ac8abbf62c44bfc819 Mon Sep 17 00:00:00 2001 From: Christian Mauderer Date: Tue, 6 Feb 2018 16:28:28 +0100 Subject: bsp/atsam: Fix cache / DMA handling in SPI. This patch fixes the cache handling for the atsam SPI driver. Note that this solution might doesn't have the best performance for small packets. --- bsps/arm/atsam/headers.am | 1 + bsps/arm/atsam/include/bsp/iocopy.h | 37 ++++ c/src/lib/libbsp/arm/atsam/Makefile.am | 3 + .../arm/atsam/libraries/libchip/source/qspi.c | 36 +--- c/src/lib/libbsp/arm/atsam/spi/atsam_spi_bus.c | 205 +++++++++++++++++++-- c/src/lib/libbsp/arm/atsam/utils/iocopy.c | 49 +++++ 6 files changed, 287 insertions(+), 44 deletions(-) create mode 100644 bsps/arm/atsam/include/bsp/iocopy.h create mode 100644 c/src/lib/libbsp/arm/atsam/utils/iocopy.c diff --git a/bsps/arm/atsam/headers.am b/bsps/arm/atsam/headers.am index 1ec34a2e69..70473c7ea1 100644 --- a/bsps/arm/atsam/headers.am +++ b/bsps/arm/atsam/headers.am @@ -11,6 +11,7 @@ include_bsp_HEADERS += ../../../../../../bsps/arm/atsam/include/bsp/atsam-clock- include_bsp_HEADERS += ../../../../../../bsps/arm/atsam/include/bsp/atsam-i2c.h include_bsp_HEADERS += ../../../../../../bsps/arm/atsam/include/bsp/atsam-spi.h include_bsp_HEADERS += ../../../../../../bsps/arm/atsam/include/bsp/i2c.h +include_bsp_HEADERS += ../../../../../../bsps/arm/atsam/include/bsp/iocopy.h include_bsp_HEADERS += ../../../../../../bsps/arm/atsam/include/bsp/irq.h include_bsp_HEADERS += ../../../../../../bsps/arm/atsam/include/bsp/pin-config.h include_bsp_HEADERS += ../../../../../../bsps/arm/atsam/include/bsp/power.h diff --git a/bsps/arm/atsam/include/bsp/iocopy.h b/bsps/arm/atsam/include/bsp/iocopy.h new file mode 100644 index 0000000000..27e374747e --- /dev/null +++ b/bsps/arm/atsam/include/bsp/iocopy.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2018 embedded brains GmbH. All rights reserved. + * + * embedded brains GmbH + * Dornierstr. 4 + * 82178 Puchheim + * Germany + * + * + * The license and distribution terms for this file may be + * found in the file LICENSE in this distribution or at + * http://www.rtems.org/license/LICENSE. + */ + +#ifndef ATSAM_IOCOPY_H +#define ATSAM_IOCOPY_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +/* + * Note: This functions are for copying from or to memory that is marked as + * Peripheral memory. In this regions a misaligned access is not allowed. + * Therefore memcopy would not work in all cases. + */ +void atsam_copy_to_io(void *dst, const void *src, size_t n); +void atsam_copy_from_io(void *dst, const void *src, size_t n); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* ATSAM_IOCOPY_H */ diff --git a/c/src/lib/libbsp/arm/atsam/Makefile.am b/c/src/lib/libbsp/arm/atsam/Makefile.am index 10f34dab55..aa97025568 100644 --- a/c/src/lib/libbsp/arm/atsam/Makefile.am +++ b/c/src/lib/libbsp/arm/atsam/Makefile.am @@ -156,6 +156,9 @@ libbsp_a_SOURCES += spi/sc16is752.c libbsp_a_SOURCES += ../../shared/tod.c libbsp_a_SOURCES += rtc/rtc-config.c +# Helper functions +libbsp_a_SOURCES += utils/iocopy.c + # Includes libbsp_a_CPPFLAGS += -I$(srcdir)/../shared/CMSIS/Include libbsp_a_CPPFLAGS += -I$(srcdir)/libraries/libboard diff --git a/c/src/lib/libbsp/arm/atsam/libraries/libchip/source/qspi.c b/c/src/lib/libbsp/arm/atsam/libraries/libchip/source/qspi.c index c80d0dd53a..314cfdf16c 100644 --- a/c/src/lib/libbsp/arm/atsam/libraries/libchip/source/qspi.c +++ b/c/src/lib/libbsp/arm/atsam/libraries/libchip/source/qspi.c @@ -75,6 +75,7 @@ #include "string.h" #include +#include #define SCRAMBLE_KEY 0x0BADDEAD @@ -211,35 +212,6 @@ __STATIC_INLINE void QSPI_ScrambleData(Qspi *pQspi, uint32_t wKey, pQspi->QSPI_SMR = (EnableFlag | (Random << 1)); } -static void do_copy(uint8_t *dst, const uint8_t *src, size_t n, bool aligned) -{ - if (aligned) { - while (n > 3) { - *(uint32_t *)dst = *(uint32_t *)src; - dst += 4; - src += 4; - n -= 4; - } - } - - while (n > 0) { - *dst = *src; - ++dst; - ++src; - --n; - } -} - -static void copy_to_io(void *dst, const void *src, size_t n) -{ - do_copy(dst, src, n, ((uintptr_t)dst) % 4 == 0); -} - -static void copy_from_io(void *dst, const void *src, size_t n) -{ - do_copy(dst, src, n, ((uintptr_t)src) % 4 == 0); -} - /*---------------------------------------------------------------------------- * Exported functions *----------------------------------------------------------------------------*/ @@ -766,9 +738,11 @@ QspidStatus_t QSPI_ReadWriteMem(Qspid_t *pQspid, Access_t const ReadWrite) && (ReadWrite <= WriteAccess)) ? true : false); if (ReadWrite == WriteAccess) { - copy_to_io(pQspiMem, pBuffer.pDataTx , pBuffer.TxDataSize); + atsam_copy_to_io(pQspiMem, pBuffer.pDataTx , + pBuffer.TxDataSize); } else { - copy_from_io(pBuffer.pDataRx, pQspiMem, pBuffer.RxDataSize); + atsam_copy_from_io(pBuffer.pDataRx, pQspiMem, + pBuffer.RxDataSize); } memory_sync(); QSPI_EndTransfer(pQspid->pQspiHw); diff --git a/c/src/lib/libbsp/arm/atsam/spi/atsam_spi_bus.c b/c/src/lib/libbsp/arm/atsam/spi/atsam_spi_bus.c index 24850c8327..35f44f525d 100644 --- a/c/src/lib/libbsp/arm/atsam/spi/atsam_spi_bus.c +++ b/c/src/lib/libbsp/arm/atsam/spi/atsam_spi_bus.c @@ -30,6 +30,7 @@ #include #include +#include #include @@ -37,16 +38,33 @@ #define MAX_SPI_FREQUENCY 50000000 +#define DMA_NR_DESC_PER_DIR 3 +#define DMA_DESC_ALLIGNMENT 4 + +#define DMA_BUF_RX 0 +#define DMA_BUF_TX 1 +#define DMA_BUF_DIRS 2 + +struct atsam_spi_xdma_buf { + LinkedListDescriporView0 desc[DMA_NR_DESC_PER_DIR]; + uint8_t leadbuf[CPU_CACHE_LINE_BYTES]; + uint8_t trailbuf[CPU_CACHE_LINE_BYTES]; +}; + typedef struct { spi_bus base; bool msg_cs_change; const spi_ioc_transfer *msg_current; + const spi_ioc_transfer *msg_next; uint32_t msg_todo; int msg_error; rtems_id msg_task; Spid spi; uint32_t dma_tx_channel; uint32_t dma_rx_channel; + struct atsam_spi_xdma_buf *dma_bufs; + size_t leadbuf_rx_buffered_len; + size_t trailbuf_rx_buffered_len; int transfer_in_progress; bool chip_select_active; bool chip_select_decode; @@ -127,17 +145,155 @@ static void atsam_configure_spi(atsam_spi_bus *bus) SPI_ConfigureNPCS(bus->spi.pSpiHw, cs, csr); } +static void atsam_spi_check_alignment_and_set_up_dma_descriptors( + atsam_spi_bus *bus, + struct atsam_spi_xdma_buf *buf, + const uint8_t *start, + size_t len, + bool tx +) +{ + LinkedListDescriporView0 *curdesc = buf->desc; + size_t misaligned_begin; + size_t misaligned_end; + size_t len_main; + const uint8_t *start_main; + const uint8_t *start_trail; + + /* Check alignments. */ + if (len < CPU_CACHE_LINE_BYTES) { + misaligned_begin = len; + misaligned_end = 0; + len_main = 0; + } else { + misaligned_begin = ((uint32_t) start) % CPU_CACHE_LINE_BYTES; + misaligned_end = (((uint32_t) start) + len) % CPU_CACHE_LINE_BYTES; + len_main = len - misaligned_begin - misaligned_end; + } + start_main = start + misaligned_begin; + start_trail = start_main + len_main; + + /* Store length for copying data back. */ + if (!tx) { + bus->leadbuf_rx_buffered_len = misaligned_begin; + bus->trailbuf_rx_buffered_len = misaligned_end; + } + + /* Handle misalignment on begin. */ + if (misaligned_begin != 0) { + if (tx) { + atsam_copy_to_io(buf->leadbuf, start, misaligned_begin); + } + curdesc->mbr_nda = (uint32_t) (&curdesc[1]); + curdesc->mbr_ta = (uint32_t) buf->leadbuf; + curdesc->mbr_ubc = misaligned_begin; + } + + /* Main part */ + if (len_main > 0) { + curdesc->mbr_ubc |= tx ? XDMA_UBC_NSEN_UPDATED : XDMA_UBC_NDEN_UPDATED; + curdesc->mbr_ubc |= XDMA_UBC_NVIEW_NDV0; + curdesc->mbr_ubc |= XDMA_UBC_NDE_FETCH_EN; + ++curdesc; + + curdesc->mbr_nda = (uint32_t) (&curdesc[1]); + curdesc->mbr_ta = (uint32_t) start_main; + curdesc->mbr_ubc = len_main; + if (tx) { + rtems_cache_flush_multiple_data_lines(start_main, len_main); + } else { + rtems_cache_invalidate_multiple_data_lines(start_main, len_main); + } + } + + /* Handle misalignment on end */ + if (misaligned_end != 0) { + curdesc->mbr_ubc |= tx ? XDMA_UBC_NSEN_UPDATED : XDMA_UBC_NDEN_UPDATED; + curdesc->mbr_ubc |= XDMA_UBC_NVIEW_NDV0; + curdesc->mbr_ubc |= XDMA_UBC_NDE_FETCH_EN; + ++curdesc; + + if (tx) { + atsam_copy_to_io(buf->trailbuf, start_trail, misaligned_end); + } + curdesc->mbr_nda = 0; + curdesc->mbr_ta = (uint32_t) buf->trailbuf; + curdesc->mbr_ubc = misaligned_end; + curdesc->mbr_ubc |= XDMA_UBC_NDE_FETCH_DIS; + } +} + +static void atsam_spi_copy_back_rx_after_dma_transfer( + atsam_spi_bus *bus +) +{ + if (bus->leadbuf_rx_buffered_len != 0) { + atsam_copy_from_io( + bus->msg_current->rx_buf, + bus->dma_bufs[DMA_BUF_RX].leadbuf, + bus->leadbuf_rx_buffered_len + ); + } + if (bus->trailbuf_rx_buffered_len != 0) { + atsam_copy_from_io( + bus->msg_current->rx_buf + bus->msg_current->len - + bus->trailbuf_rx_buffered_len, + bus->dma_bufs[DMA_BUF_RX].trailbuf, + bus->trailbuf_rx_buffered_len + ); + } +} + static void atsam_spi_start_dma_transfer( atsam_spi_bus *bus, const spi_ioc_transfer *msg ) { Xdmac *pXdmac = XDMAC; + size_t i; + + atsam_spi_check_alignment_and_set_up_dma_descriptors( + bus, + &bus->dma_bufs[DMA_BUF_RX], + msg->rx_buf, + msg->len, + false + ); + atsam_spi_check_alignment_and_set_up_dma_descriptors( + bus, + &bus->dma_bufs[DMA_BUF_TX], + msg->tx_buf, + msg->len, + true + ); + + XDMAC_SetDescriptorAddr( + pXdmac, + bus->dma_rx_channel, + (uint32_t) bus->dma_bufs[DMA_BUF_RX].desc, + 0 + ); + XDMAC_SetDescriptorControl( + pXdmac, + bus->dma_rx_channel, + XDMAC_CNDC_NDVIEW_NDV0 | + XDMAC_CNDC_NDDUP_DST_PARAMS_UPDATED | + XDMAC_CNDC_NDE_DSCR_FETCH_EN + ); + XDMAC_SetDescriptorAddr( + pXdmac, + bus->dma_tx_channel, + (uint32_t) bus->dma_bufs[DMA_BUF_TX].desc, + 0 + ); + XDMAC_SetDescriptorControl( + pXdmac, + bus->dma_tx_channel, + XDMAC_CNDC_NDVIEW_NDV0 | + XDMAC_CNDC_NDSUP_SRC_PARAMS_UPDATED | + XDMAC_CNDC_NDE_DSCR_FETCH_EN + ); - XDMAC_SetDestinationAddr(pXdmac, bus->dma_rx_channel, (uint32_t)msg->rx_buf); - XDMAC_SetSourceAddr(pXdmac, bus->dma_tx_channel, (uint32_t)msg->tx_buf); - XDMAC_SetMicroblockControl(pXdmac, bus->dma_rx_channel, msg->len); - XDMAC_SetMicroblockControl(pXdmac, bus->dma_tx_channel, msg->len); XDMAC_StartTransfer(pXdmac, bus->dma_rx_channel); XDMAC_StartTransfer(pXdmac, bus->dma_tx_channel); } @@ -205,11 +361,12 @@ static void atsam_spi_setup_transfer(atsam_spi_bus *bus) } if (msg_todo > 0) { - const spi_ioc_transfer *msg = bus->msg_current; + const spi_ioc_transfer *msg = bus->msg_next; int error; bus->msg_cs_change = msg->cs_change; - bus->msg_current = msg + 1; + bus->msg_next = msg + 1; + bus->msg_current = msg; bus->msg_todo = msg_todo - 1; error = atsam_check_configure_spi(bus, msg); @@ -231,6 +388,7 @@ static void atsam_spi_dma_callback(uint32_t channel, void *arg) --bus->transfer_in_progress; if (bus->transfer_in_progress == 0) { + atsam_spi_copy_back_rx_after_dma_transfer(bus); atsam_spi_setup_transfer(bus); } } @@ -241,15 +399,18 @@ static int atsam_spi_transfer( uint32_t msg_count ) { + rtems_status_code sc; atsam_spi_bus *bus = (atsam_spi_bus *)base; bus->msg_cs_change = false; - bus->msg_current = &msgs[0]; + bus->msg_next = &msgs[0]; + bus->msg_current = NULL; bus->msg_todo = msg_count; bus->msg_error = 0; bus->msg_task = rtems_task_self(); atsam_spi_setup_transfer(bus); - rtems_event_transient_receive(RTEMS_WAIT, RTEMS_NO_TIMEOUT); + sc = rtems_event_transient_receive(RTEMS_WAIT, RTEMS_NO_TIMEOUT); + assert(sc == RTEMS_SUCCESSFUL); return bus->msg_error; } @@ -281,6 +442,8 @@ static void atsam_spi_destroy(spi_bus *base) SPI_Disable(bus->spi.pSpiHw); PMC_DisablePeripheral(bus->spi.spiId); + rtems_cache_coherent_free(bus->dma_bufs); + spi_bus_destroy_and_free(&bus->base); } @@ -305,6 +468,14 @@ static void atsam_spi_init_xdma(atsam_spi_bus *bus) uint32_t xdmaInt; uint8_t channel; eXdmadRC rc; + uint32_t xdma_cndc; + + bus->dma_bufs = rtems_cache_coherent_allocate( + DMA_BUF_DIRS * sizeof(*(bus->dma_bufs)), + DMA_DESC_ALLIGNMENT, + 0 + ); + assert(bus->dma_bufs != NULL); bus->dma_tx_channel = XDMAD_AllocateChannel( bus->spi.pXdmad, @@ -342,7 +513,7 @@ static void atsam_spi_init_xdma(atsam_spi_bus *bus) rc = XDMAD_PrepareChannel(bus->spi.pXdmad, bus->dma_tx_channel); assert(rc == XDMAD_OK); - /* Put all interrupts on for non LLI list setup of DMA */ + /* Put all relevant interrupts on */ xdmaInt = ( XDMAC_CIE_BIE | XDMAC_CIE_DIE | @@ -366,12 +537,16 @@ static void atsam_spi_init_xdma(atsam_spi_bus *bus) XDMAC_CC_SAM_FIXED_AM | XDMAC_CC_DAM_INCREMENTED_AM | XDMAC_CC_PERID(channel); + xdma_cndc = XDMAC_CNDC_NDVIEW_NDV0 | + XDMAC_CNDC_NDE_DSCR_FETCH_EN | + XDMAC_CNDC_NDDUP_DST_PARAMS_UPDATED | + XDMAC_CNDC_NDSUP_SRC_PARAMS_UNCHANGED; rc = XDMAD_ConfigureTransfer( bus->spi.pXdmad, bus->dma_rx_channel, &cfg, - 0, - 0, + xdma_cndc, + (uint32_t) bus->dma_bufs[DMA_BUF_RX].desc, xdmaInt ); assert(rc == XDMAD_OK); @@ -391,12 +566,16 @@ static void atsam_spi_init_xdma(atsam_spi_bus *bus) XDMAC_CC_SAM_INCREMENTED_AM | XDMAC_CC_DAM_FIXED_AM | XDMAC_CC_PERID(channel); + xdma_cndc = XDMAC_CNDC_NDVIEW_NDV0 | + XDMAC_CNDC_NDE_DSCR_FETCH_EN | + XDMAC_CNDC_NDDUP_DST_PARAMS_UNCHANGED | + XDMAC_CNDC_NDSUP_SRC_PARAMS_UPDATED; rc = XDMAD_ConfigureTransfer( bus->spi.pXdmad, bus->dma_tx_channel, &cfg, - 0, - 0, + xdma_cndc, + (uint32_t) bus->dma_bufs[DMA_BUF_TX].desc, xdmaInt ); assert(rc == XDMAD_OK); diff --git a/c/src/lib/libbsp/arm/atsam/utils/iocopy.c b/c/src/lib/libbsp/arm/atsam/utils/iocopy.c new file mode 100644 index 0000000000..b91282dac6 --- /dev/null +++ b/c/src/lib/libbsp/arm/atsam/utils/iocopy.c @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2018 embedded brains GmbH. All rights reserved. + * + * embedded brains GmbH + * Dornierstr. 4 + * 82178 Puchheim + * Germany + * + * + * The license and distribution terms for this file may be + * found in the file LICENSE in this distribution or at + * http://www.rtems.org/license/LICENSE. + */ + +#include + +static void atsam_do_copy( + uint8_t *dst, + const uint8_t *src, + size_t n, + bool aligned +) +{ + if (aligned) { + while (n > 3) { + *(uint32_t *)dst = *(uint32_t *)src; + dst += 4; + src += 4; + n -= 4; + } + } + + while (n > 0) { + *dst = *src; + ++dst; + ++src; + --n; + } +} + +void atsam_copy_to_io(void *dst, const void *src, size_t n) +{ + atsam_do_copy(dst, src, n, ((uintptr_t)dst) % 4 == 0); +} + +void atsam_copy_from_io(void *dst, const void *src, size_t n) +{ + atsam_do_copy(dst, src, n, ((uintptr_t)src) % 4 == 0); +} -- cgit v1.2.3