summaryrefslogtreecommitdiffstats
path: root/linux/drivers/soc/fsl/qbman/qman.h
diff options
context:
space:
mode:
Diffstat (limited to 'linux/drivers/soc/fsl/qbman/qman.h')
-rw-r--r--linux/drivers/soc/fsl/qbman/qman.h1133
1 files changed, 0 insertions, 1133 deletions
diff --git a/linux/drivers/soc/fsl/qbman/qman.h b/linux/drivers/soc/fsl/qbman/qman.h
deleted file mode 100644
index 331db7c7..00000000
--- a/linux/drivers/soc/fsl/qbman/qman.h
+++ /dev/null
@@ -1,1133 +0,0 @@
-/* Copyright 2008 - 2015 Freescale Semiconductor, Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * * Neither the name of Freescale Semiconductor nor the
- * names of its contributors may be used to endorse or promote products
- * derived from this software without specific prior written permission.
- *
- * ALTERNATIVELY, this software may be distributed under the terms of the
- * GNU General Public License ("GPL") as published by the Free Software
- * Foundation, either version 2 of that License or (at your option) any
- * later version.
- *
- * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "qman_priv.h"
-
-/* Portal register assists */
-
-/* Cache-inhibited register offsets */
-#define QM_REG_EQCR_PI_CINH 0x0000
-#define QM_REG_EQCR_CI_CINH 0x0004
-#define QM_REG_EQCR_ITR 0x0008
-#define QM_REG_DQRR_PI_CINH 0x0040
-#define QM_REG_DQRR_CI_CINH 0x0044
-#define QM_REG_DQRR_ITR 0x0048
-#define QM_REG_DQRR_DCAP 0x0050
-#define QM_REG_DQRR_SDQCR 0x0054
-#define QM_REG_DQRR_VDQCR 0x0058
-#define QM_REG_DQRR_PDQCR 0x005c
-#define QM_REG_MR_PI_CINH 0x0080
-#define QM_REG_MR_CI_CINH 0x0084
-#define QM_REG_MR_ITR 0x0088
-#define QM_REG_CFG 0x0100
-#define QM_REG_ISR 0x0e00
-#define QM_REG_IIR 0x0e0c
-#define QM_REG_ITPR 0x0e14
-
-/* Cache-enabled register offsets */
-#define QM_CL_EQCR 0x0000
-#define QM_CL_DQRR 0x1000
-#define QM_CL_MR 0x2000
-#define QM_CL_EQCR_PI_CENA 0x3000
-#define QM_CL_EQCR_CI_CENA 0x3100
-#define QM_CL_DQRR_PI_CENA 0x3200
-#define QM_CL_DQRR_CI_CENA 0x3300
-#define QM_CL_MR_PI_CENA 0x3400
-#define QM_CL_MR_CI_CENA 0x3500
-#define QM_CL_CR 0x3800
-#define QM_CL_RR0 0x3900
-#define QM_CL_RR1 0x3940
-
-/* BTW, the drivers (and h/w programming model) already obtain the required
- * synchronisation for portal accesses via lwsync(), hwsync(), and
- * data-dependencies. Use of barrier()s or other order-preserving primitives
- * simply degrade performance. Hence the use of the __raw_*() interfaces, which
- * simply ensure that the compiler treats the portal registers as volatile (ie.
- * non-coherent). */
-
-/* Cache-inhibited register access. */
-#define __qm_in(qm, o) __raw_readl((qm)->addr_ci + (o))
-#define __qm_out(qm, o, val) __raw_writel((val), (qm)->addr_ci + (o))
-#define qm_in(reg) __qm_in(&portal->addr, QM_REG_##reg)
-#define qm_out(reg, val) __qm_out(&portal->addr, QM_REG_##reg, val)
-
-/* Cache-enabled (index) register access */
-#define __qm_cl_touch_ro(qm, o) dcbt_ro((qm)->addr_ce + (o))
-#define __qm_cl_touch_rw(qm, o) dcbt_rw((qm)->addr_ce + (o))
-#define __qm_cl_in(qm, o) __raw_readl((qm)->addr_ce + (o))
-#define __qm_cl_out(qm, o, val) \
- do { \
- u32 *__tmpclout = (qm)->addr_ce + (o); \
- __raw_writel((val), __tmpclout); \
- dcbf(__tmpclout); \
- } while (0)
-#define __qm_cl_invalidate(qm, o) dcbi((qm)->addr_ce + (o))
-#define qm_cl_touch_ro(reg) __qm_cl_touch_ro(&portal->addr, QM_CL_##reg##_CENA)
-#define qm_cl_touch_rw(reg) __qm_cl_touch_rw(&portal->addr, QM_CL_##reg##_CENA)
-#define qm_cl_in(reg) __qm_cl_in(&portal->addr, QM_CL_##reg##_CENA)
-#define qm_cl_out(reg, val) __qm_cl_out(&portal->addr, QM_CL_##reg##_CENA, val)
-#define qm_cl_invalidate(reg)\
- __qm_cl_invalidate(&portal->addr, QM_CL_##reg##_CENA)
-
-/* Cache-enabled ring access */
-#define qm_cl(base, idx) ((void *)base + ((idx) << 6))
-
-/* Cyclic helper for rings. FIXME: once we are able to do fine-grain perf
- * analysis, look at using the "extra" bit in the ring index registers to avoid
- * cyclic issues. */
-static inline u8 qm_cyc_diff(u8 ringsize, u8 first, u8 last)
-{
- /* 'first' is included, 'last' is excluded */
- if (first <= last)
- return last - first;
- return ringsize + last - first;
-}
-
-/* Portal modes.
- * Enum types;
- * pmode == production mode
- * cmode == consumption mode,
- * dmode == h/w dequeue mode.
- * Enum values use 3 letter codes. First letter matches the portal mode,
- * remaining two letters indicate;
- * ci == cache-inhibited portal register
- * ce == cache-enabled portal register
- * vb == in-band valid-bit (cache-enabled)
- * dc == DCA (Discrete Consumption Acknowledgment), DQRR-only
- * As for "enum qm_dqrr_dmode", it should be self-explanatory.
- */
-enum qm_eqcr_pmode { /* matches QCSP_CFG::EPM */
- qm_eqcr_pci = 0, /* PI index, cache-inhibited */
- qm_eqcr_pce = 1, /* PI index, cache-enabled */
- qm_eqcr_pvb = 2 /* valid-bit */
-};
-enum qm_dqrr_dmode { /* matches QCSP_CFG::DP */
- qm_dqrr_dpush = 0, /* SDQCR + VDQCR */
- qm_dqrr_dpull = 1 /* PDQCR */
-};
-enum qm_dqrr_pmode { /* s/w-only */
- qm_dqrr_pci, /* reads DQRR_PI_CINH */
- qm_dqrr_pce, /* reads DQRR_PI_CENA */
- qm_dqrr_pvb /* reads valid-bit */
-};
-enum qm_dqrr_cmode { /* matches QCSP_CFG::DCM */
- qm_dqrr_cci = 0, /* CI index, cache-inhibited */
- qm_dqrr_cce = 1, /* CI index, cache-enabled */
- qm_dqrr_cdc = 2 /* Discrete Consumption Acknowledgment */
-};
-enum qm_mr_pmode { /* s/w-only */
- qm_mr_pci, /* reads MR_PI_CINH */
- qm_mr_pce, /* reads MR_PI_CENA */
- qm_mr_pvb /* reads valid-bit */
-};
-enum qm_mr_cmode { /* matches QCSP_CFG::MM */
- qm_mr_cci = 0, /* CI index, cache-inhibited */
- qm_mr_cce = 1 /* CI index, cache-enabled */
-};
-
-/* --- Portal structures --- */
-
-#define QM_EQCR_SIZE 8
-#define QM_DQRR_SIZE 16
-#define QM_MR_SIZE 8
-
-struct qm_eqcr {
- struct qm_eqcr_entry *ring, *cursor;
- u8 ci, available, ithresh, vbit;
-#ifdef CONFIG_FSL_DPA_CHECKING
- u32 busy;
- enum qm_eqcr_pmode pmode;
-#endif
-};
-
-struct qm_dqrr {
- const struct qm_dqrr_entry *ring, *cursor;
- u8 pi, ci, fill, ithresh, vbit;
-#ifdef CONFIG_FSL_DPA_CHECKING
- enum qm_dqrr_dmode dmode;
- enum qm_dqrr_pmode pmode;
- enum qm_dqrr_cmode cmode;
-#endif
-};
-
-struct qm_mr {
- const struct qm_mr_entry *ring, *cursor;
- u8 pi, ci, fill, ithresh, vbit;
-#ifdef CONFIG_FSL_DPA_CHECKING
- enum qm_mr_pmode pmode;
- enum qm_mr_cmode cmode;
-#endif
-};
-
-struct qm_mc {
- struct qm_mc_command *cr;
- struct qm_mc_result *rr;
- u8 rridx, vbit;
-#ifdef CONFIG_FSL_DPA_CHECKING
- enum {
- /* Can be _mc_start()ed */
- qman_mc_idle,
- /* Can be _mc_commit()ed or _mc_abort()ed */
- qman_mc_user,
- /* Can only be _mc_retry()ed */
- qman_mc_hw
- } state;
-#endif
-};
-
-#define QM_PORTAL_ALIGNMENT ____cacheline_aligned
-
-struct qm_addr {
- void __iomem *addr_ce; /* cache-enabled */
- void __iomem *addr_ci; /* cache-inhibited */
-};
-
-struct qm_portal {
- /* In the non-CONFIG_FSL_DPA_CHECKING case, the following stuff up to
- * and including 'mc' fits within a cacheline (yay!). The 'config' part
- * is setup-only, so isn't a cause for a concern. In other words, don't
- * rearrange this structure on a whim, there be dragons ... */
- struct qm_addr addr;
- struct qm_eqcr eqcr;
- struct qm_dqrr dqrr;
- struct qm_mr mr;
- struct qm_mc mc;
-} QM_PORTAL_ALIGNMENT;
-
-/* --- EQCR API --- */
-
-/* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */
-#define EQCR_CARRYCLEAR(p) \
- (void *)((unsigned long)(p) & (~(unsigned long)(QM_EQCR_SIZE << 6)))
-
-/* Bit-wise logic to convert a ring pointer to a ring index */
-static inline u8 EQCR_PTR2IDX(struct qm_eqcr_entry *e)
-{
- return ((uintptr_t)e >> 6) & (QM_EQCR_SIZE - 1);
-}
-
-/* Increment the 'cursor' ring pointer, taking 'vbit' into account */
-static inline void EQCR_INC(struct qm_eqcr *eqcr)
-{
- /* NB: this is odd-looking, but experiments show that it generates fast
- * code with essentially no branching overheads. We increment to the
- * next EQCR pointer and handle overflow and 'vbit'. */
- struct qm_eqcr_entry *partial = eqcr->cursor + 1;
-
- eqcr->cursor = EQCR_CARRYCLEAR(partial);
- if (partial != eqcr->cursor)
- eqcr->vbit ^= QM_EQCR_VERB_VBIT;
-}
-
-static inline int qm_eqcr_init(struct qm_portal *portal,
- enum qm_eqcr_pmode pmode,
- unsigned int eq_stash_thresh,
- int eq_stash_prio)
-{
- /* This use of 'register', as well as all other occurrences, is because
- * it has been observed to generate much faster code with gcc than is
- * otherwise the case. */
- register struct qm_eqcr *eqcr = &portal->eqcr;
- u32 cfg;
- u8 pi;
-
- eqcr->ring = portal->addr.addr_ce + QM_CL_EQCR;
- eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
- qm_cl_invalidate(EQCR_CI);
- pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
- eqcr->cursor = eqcr->ring + pi;
- eqcr->vbit = (qm_in(EQCR_PI_CINH) & QM_EQCR_SIZE) ?
- QM_EQCR_VERB_VBIT : 0;
- eqcr->available = QM_EQCR_SIZE - 1 -
- qm_cyc_diff(QM_EQCR_SIZE, eqcr->ci, pi);
- eqcr->ithresh = qm_in(EQCR_ITR);
-#ifdef CONFIG_FSL_DPA_CHECKING
- eqcr->busy = 0;
- eqcr->pmode = pmode;
-#endif
- cfg = (qm_in(CFG) & 0x00ffffff) |
- (eq_stash_thresh << 28) | /* QCSP_CFG: EST */
- (eq_stash_prio << 26) | /* QCSP_CFG: EP */
- ((pmode & 0x3) << 24); /* QCSP_CFG::EPM */
- qm_out(CFG, cfg);
- return 0;
-}
-
-static inline unsigned int qm_eqcr_get_ci_stashing(struct qm_portal *portal)
-{
- return (qm_in(CFG) >> 28) & 0x7;
-}
-
-static inline void qm_eqcr_finish(struct qm_portal *portal)
-{
- register struct qm_eqcr *eqcr = &portal->eqcr;
- u8 pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
- u8 ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
-
- DPA_ASSERT(!eqcr->busy);
- if (pi != EQCR_PTR2IDX(eqcr->cursor))
- pr_crit("losing uncommited EQCR entries\n");
- if (ci != eqcr->ci)
- pr_crit("missing existing EQCR completions\n");
- if (eqcr->ci != EQCR_PTR2IDX(eqcr->cursor))
- pr_crit("EQCR destroyed unquiesced\n");
-}
-
-static inline struct qm_eqcr_entry *qm_eqcr_start_no_stash(struct qm_portal
- *portal)
-{
- register struct qm_eqcr *eqcr = &portal->eqcr;
-
- DPA_ASSERT(!eqcr->busy);
- if (!eqcr->available)
- return NULL;
-
-
-#ifdef CONFIG_FSL_DPA_CHECKING
- eqcr->busy = 1;
-#endif
- dcbz_64(eqcr->cursor);
- return eqcr->cursor;
-}
-
-static inline struct qm_eqcr_entry *qm_eqcr_start_stash(struct qm_portal
- *portal)
-{
- register struct qm_eqcr *eqcr = &portal->eqcr;
- u8 diff, old_ci;
-
- DPA_ASSERT(!eqcr->busy);
- if (!eqcr->available) {
- old_ci = eqcr->ci;
- eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
- diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
- eqcr->available += diff;
- if (!diff)
- return NULL;
- }
-#ifdef CONFIG_FSL_DPA_CHECKING
- eqcr->busy = 1;
-#endif
- dcbz_64(eqcr->cursor);
- return eqcr->cursor;
-}
-
-static inline void qm_eqcr_abort(struct qm_portal *portal)
-{
- __maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr;
-
- DPA_ASSERT(eqcr->busy);
-#ifdef CONFIG_FSL_DPA_CHECKING
- eqcr->busy = 0;
-#endif
-}
-
-static inline struct qm_eqcr_entry *qm_eqcr_pend_and_next(
- struct qm_portal *portal, u8 myverb)
-{
- register struct qm_eqcr *eqcr = &portal->eqcr;
-
- DPA_ASSERT(eqcr->busy);
- DPA_ASSERT(eqcr->pmode != qm_eqcr_pvb);
- if (eqcr->available == 1)
- return NULL;
- eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit;
- dcbf(eqcr->cursor);
- EQCR_INC(eqcr);
- eqcr->available--;
- dcbz_64(eqcr->cursor);
- return eqcr->cursor;
-}
-
-#define EQCR_COMMIT_CHECKS(eqcr) \
-do { \
- DPA_ASSERT(eqcr->busy); \
- DPA_ASSERT(eqcr->cursor->orp == (eqcr->cursor->orp & 0x00ffffff)); \
- DPA_ASSERT(eqcr->cursor->fqid == (eqcr->cursor->fqid & 0x00ffffff)); \
-} while (0)
-
-static inline void qm_eqcr_pci_commit(struct qm_portal *portal, u8 myverb)
-{
- register struct qm_eqcr *eqcr = &portal->eqcr;
-
- EQCR_COMMIT_CHECKS(eqcr);
- DPA_ASSERT(eqcr->pmode == qm_eqcr_pci);
- eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit;
- EQCR_INC(eqcr);
- eqcr->available--;
- dcbf(eqcr->cursor);
- hwsync();
- qm_out(EQCR_PI_CINH, EQCR_PTR2IDX(eqcr->cursor));
-#ifdef CONFIG_FSL_DPA_CHECKING
- eqcr->busy = 0;
-#endif
-}
-
-static inline void qm_eqcr_pce_prefetch(struct qm_portal *portal)
-{
- __maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr;
-
- DPA_ASSERT(eqcr->pmode == qm_eqcr_pce);
- qm_cl_invalidate(EQCR_PI);
- qm_cl_touch_rw(EQCR_PI);
-}
-
-static inline void qm_eqcr_pce_commit(struct qm_portal *portal, u8 myverb)
-{
- register struct qm_eqcr *eqcr = &portal->eqcr;
-
- EQCR_COMMIT_CHECKS(eqcr);
- DPA_ASSERT(eqcr->pmode == qm_eqcr_pce);
- eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit;
- EQCR_INC(eqcr);
- eqcr->available--;
- dcbf(eqcr->cursor);
- lwsync();
- qm_cl_out(EQCR_PI, EQCR_PTR2IDX(eqcr->cursor));
-#ifdef CONFIG_FSL_DPA_CHECKING
- eqcr->busy = 0;
-#endif
-}
-
-static inline void qm_eqcr_pvb_commit(struct qm_portal *portal, u8 myverb)
-{
- register struct qm_eqcr *eqcr = &portal->eqcr;
- struct qm_eqcr_entry *eqcursor;
-
- EQCR_COMMIT_CHECKS(eqcr);
- DPA_ASSERT(eqcr->pmode == qm_eqcr_pvb);
- lwsync();
- eqcursor = eqcr->cursor;
- eqcursor->__dont_write_directly__verb = myverb | eqcr->vbit;
- dcbf(eqcursor);
- EQCR_INC(eqcr);
- eqcr->available--;
-#ifdef CONFIG_FSL_DPA_CHECKING
- eqcr->busy = 0;
-#endif
-}
-
-static inline u8 qm_eqcr_cci_update(struct qm_portal *portal)
-{
- register struct qm_eqcr *eqcr = &portal->eqcr;
- u8 diff, old_ci = eqcr->ci;
-
- eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
- diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
- eqcr->available += diff;
- return diff;
-}
-
-static inline void qm_eqcr_cce_prefetch(struct qm_portal *portal)
-{
- __maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr;
-
- qm_cl_touch_ro(EQCR_CI);
-}
-
-static inline u8 qm_eqcr_cce_update(struct qm_portal *portal)
-{
- register struct qm_eqcr *eqcr = &portal->eqcr;
- u8 diff, old_ci = eqcr->ci;
-
- eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
- qm_cl_invalidate(EQCR_CI);
- diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
- eqcr->available += diff;
- return diff;
-}
-
-static inline u8 qm_eqcr_get_ithresh(struct qm_portal *portal)
-{
- register struct qm_eqcr *eqcr = &portal->eqcr;
-
- return eqcr->ithresh;
-}
-
-static inline void qm_eqcr_set_ithresh(struct qm_portal *portal, u8 ithresh)
-{
- register struct qm_eqcr *eqcr = &portal->eqcr;
-
- eqcr->ithresh = ithresh;
- qm_out(EQCR_ITR, ithresh);
-}
-
-static inline u8 qm_eqcr_get_avail(struct qm_portal *portal)
-{
- register struct qm_eqcr *eqcr = &portal->eqcr;
-
- return eqcr->available;
-}
-
-static inline u8 qm_eqcr_get_fill(struct qm_portal *portal)
-{
- register struct qm_eqcr *eqcr = &portal->eqcr;
-
- return QM_EQCR_SIZE - 1 - eqcr->available;
-}
-
-/* --- DQRR API --- */
-
-/* FIXME: many possible improvements;
- * - look at changing the API to use pointer rather than index parameters now
- * that 'cursor' is a pointer,
- * - consider moving other parameters to pointer if it could help (ci)
- */
-
-#define DQRR_CARRYCLEAR(p) \
- (void *)((unsigned long)(p) & (~(unsigned long)(QM_DQRR_SIZE << 6)))
-
-static inline u8 DQRR_PTR2IDX(const struct qm_dqrr_entry *e)
-{
- return ((uintptr_t)e >> 6) & (QM_DQRR_SIZE - 1);
-}
-
-static inline const struct qm_dqrr_entry *DQRR_INC(
- const struct qm_dqrr_entry *e)
-{
- return DQRR_CARRYCLEAR(e + 1);
-}
-
-static inline void qm_dqrr_set_maxfill(struct qm_portal *portal, u8 mf)
-{
- qm_out(CFG, (qm_in(CFG) & 0xff0fffff) |
- ((mf & (QM_DQRR_SIZE - 1)) << 20));
-}
-
-static inline int qm_dqrr_init(struct qm_portal *portal,
- const struct qm_portal_config *config,
- enum qm_dqrr_dmode dmode,
- __maybe_unused enum qm_dqrr_pmode pmode,
- enum qm_dqrr_cmode cmode, u8 max_fill)
-{
- register struct qm_dqrr *dqrr = &portal->dqrr;
- u32 cfg;
-
- /* Make sure the DQRR will be idle when we enable */
- qm_out(DQRR_SDQCR, 0);
- qm_out(DQRR_VDQCR, 0);
- qm_out(DQRR_PDQCR, 0);
- dqrr->ring = portal->addr.addr_ce + QM_CL_DQRR;
- dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1);
- dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);
- dqrr->cursor = dqrr->ring + dqrr->ci;
- dqrr->fill = qm_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi);
- dqrr->vbit = (qm_in(DQRR_PI_CINH) & QM_DQRR_SIZE) ?
- QM_DQRR_VERB_VBIT : 0;
- dqrr->ithresh = qm_in(DQRR_ITR);
-#ifdef CONFIG_FSL_DPA_CHECKING
- dqrr->dmode = dmode;
- dqrr->pmode = pmode;
- dqrr->cmode = cmode;
-#endif
- /* Invalidate every ring entry before beginning */
- for (cfg = 0; cfg < QM_DQRR_SIZE; cfg++)
- dcbi(qm_cl(dqrr->ring, cfg));
- cfg = (qm_in(CFG) & 0xff000f00) |
- ((max_fill & (QM_DQRR_SIZE - 1)) << 20) | /* DQRR_MF */
- ((dmode & 1) << 18) | /* DP */
- ((cmode & 3) << 16) | /* DCM */
-#ifndef __rtems__
- 0xa0 | /* RE+SE */
-#endif /* __rtems__ */
- (0 ? 0x40 : 0) | /* Ignore RP */
- (0 ? 0x10 : 0); /* Ignore SP */
- qm_out(CFG, cfg);
- qm_dqrr_set_maxfill(portal, max_fill);
- return 0;
-}
-
-static inline void qm_dqrr_finish(struct qm_portal *portal)
-{
- __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
-#ifdef CONFIG_FSL_DPA_CHECKING
- if ((dqrr->cmode != qm_dqrr_cdc) &&
- (dqrr->ci != DQRR_PTR2IDX(dqrr->cursor)))
- pr_crit("Ignoring completed DQRR entries\n");
-#endif
-}
-
-static inline const struct qm_dqrr_entry *qm_dqrr_current(
- struct qm_portal *portal)
-{
- register struct qm_dqrr *dqrr = &portal->dqrr;
-
- if (!dqrr->fill)
- return NULL;
- return dqrr->cursor;
-}
-
-static inline u8 qm_dqrr_cursor(struct qm_portal *portal)
-{
- register struct qm_dqrr *dqrr = &portal->dqrr;
-
- return DQRR_PTR2IDX(dqrr->cursor);
-}
-
-static inline u8 qm_dqrr_next(struct qm_portal *portal)
-{
- register struct qm_dqrr *dqrr = &portal->dqrr;
-
- DPA_ASSERT(dqrr->fill);
- dqrr->cursor = DQRR_INC(dqrr->cursor);
- return --dqrr->fill;
-}
-
-static inline u8 qm_dqrr_pci_update(struct qm_portal *portal)
-{
- register struct qm_dqrr *dqrr = &portal->dqrr;
- u8 diff, old_pi = dqrr->pi;
-
- DPA_ASSERT(dqrr->pmode == qm_dqrr_pci);
- dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1);
- diff = qm_cyc_diff(QM_DQRR_SIZE, old_pi, dqrr->pi);
- dqrr->fill += diff;
- return diff;
-}
-
-static inline void qm_dqrr_pce_prefetch(struct qm_portal *portal)
-{
- __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
-
- DPA_ASSERT(dqrr->pmode == qm_dqrr_pce);
- qm_cl_invalidate(DQRR_PI);
- qm_cl_touch_ro(DQRR_PI);
-}
-
-static inline u8 qm_dqrr_pce_update(struct qm_portal *portal)
-{
- register struct qm_dqrr *dqrr = &portal->dqrr;
- u8 diff, old_pi = dqrr->pi;
-
- DPA_ASSERT(dqrr->pmode == qm_dqrr_pce);
- dqrr->pi = qm_cl_in(DQRR_PI) & (QM_DQRR_SIZE - 1);
- diff = qm_cyc_diff(QM_DQRR_SIZE, old_pi, dqrr->pi);
- dqrr->fill += diff;
- return diff;
-}
-
-static inline void qm_dqrr_pvb_update(struct qm_portal *portal)
-{
- register struct qm_dqrr *dqrr = &portal->dqrr;
- const struct qm_dqrr_entry *res = qm_cl(dqrr->ring, dqrr->pi);
-
- DPA_ASSERT(dqrr->pmode == qm_dqrr_pvb);
- /* when accessing 'verb', use __raw_readb() to ensure that compiler
- * inlining doesn't try to optimise out "excess reads". */
-#ifdef __rtems__
- dcbi(res);
-#endif /* __rtems__ */
- if ((__raw_readb(&res->verb) & QM_DQRR_VERB_VBIT) == dqrr->vbit) {
- dqrr->pi = (dqrr->pi + 1) & (QM_DQRR_SIZE - 1);
- if (!dqrr->pi)
- dqrr->vbit ^= QM_DQRR_VERB_VBIT;
- dqrr->fill++;
- }
-}
-
-static inline void qm_dqrr_cci_consume(struct qm_portal *portal, u8 num)
-{
- register struct qm_dqrr *dqrr = &portal->dqrr;
-
- DPA_ASSERT(dqrr->cmode == qm_dqrr_cci);
- dqrr->ci = (dqrr->ci + num) & (QM_DQRR_SIZE - 1);
- qm_out(DQRR_CI_CINH, dqrr->ci);
-}
-
-static inline void qm_dqrr_cci_consume_to_current(struct qm_portal *portal)
-{
- register struct qm_dqrr *dqrr = &portal->dqrr;
-
- DPA_ASSERT(dqrr->cmode == qm_dqrr_cci);
- dqrr->ci = DQRR_PTR2IDX(dqrr->cursor);
- qm_out(DQRR_CI_CINH, dqrr->ci);
-}
-
-static inline void qm_dqrr_cce_prefetch(struct qm_portal *portal)
-{
- __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
-
- DPA_ASSERT(dqrr->cmode == qm_dqrr_cce);
- qm_cl_invalidate(DQRR_CI);
- qm_cl_touch_rw(DQRR_CI);
-}
-
-static inline void qm_dqrr_cce_consume(struct qm_portal *portal, u8 num)
-{
- register struct qm_dqrr *dqrr = &portal->dqrr;
-
- DPA_ASSERT(dqrr->cmode == qm_dqrr_cce);
- dqrr->ci = (dqrr->ci + num) & (QM_DQRR_SIZE - 1);
- qm_cl_out(DQRR_CI, dqrr->ci);
-}
-
-static inline void qm_dqrr_cce_consume_to_current(struct qm_portal *portal)
-{
- register struct qm_dqrr *dqrr = &portal->dqrr;
-
- DPA_ASSERT(dqrr->cmode == qm_dqrr_cce);
- dqrr->ci = DQRR_PTR2IDX(dqrr->cursor);
- qm_cl_out(DQRR_CI, dqrr->ci);
-}
-
-static inline void qm_dqrr_cdc_consume_1(struct qm_portal *portal, u8 idx,
- int park)
-{
- __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
-
- DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
- DPA_ASSERT(idx < QM_DQRR_SIZE);
- qm_out(DQRR_DCAP, (0 << 8) | /* S */
- ((park ? 1 : 0) << 6) | /* PK */
- idx); /* DCAP_CI */
-}
-
-static inline void qm_dqrr_cdc_consume_1ptr(struct qm_portal *portal,
- const struct qm_dqrr_entry *dq,
- int park)
-{
- __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
- u8 idx = DQRR_PTR2IDX(dq);
-
- DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
- DPA_ASSERT((dqrr->ring + idx) == dq);
- DPA_ASSERT(idx < QM_DQRR_SIZE);
- qm_out(DQRR_DCAP, (0 << 8) | /* DQRR_DCAP::S */
- ((park ? 1 : 0) << 6) | /* DQRR_DCAP::PK */
- idx); /* DQRR_DCAP::DCAP_CI */
-}
-
-static inline void qm_dqrr_cdc_consume_n(struct qm_portal *portal, u16 bitmask)
-{
- __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
-
- DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
- qm_out(DQRR_DCAP, (1 << 8) | /* DQRR_DCAP::S */
- ((u32)bitmask << 16)); /* DQRR_DCAP::DCAP_CI */
-}
-
-static inline u8 qm_dqrr_cdc_cci(struct qm_portal *portal)
-{
- __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
-
- DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
- return qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);
-}
-
-static inline void qm_dqrr_cdc_cce_prefetch(struct qm_portal *portal)
-{
- __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
-
- DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
- qm_cl_invalidate(DQRR_CI);
- qm_cl_touch_ro(DQRR_CI);
-}
-
-static inline u8 qm_dqrr_cdc_cce(struct qm_portal *portal)
-{
- __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
-
- DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
- return qm_cl_in(DQRR_CI) & (QM_DQRR_SIZE - 1);
-}
-
-static inline u8 qm_dqrr_get_ci(struct qm_portal *portal)
-{
- register struct qm_dqrr *dqrr = &portal->dqrr;
-
- DPA_ASSERT(dqrr->cmode != qm_dqrr_cdc);
- return dqrr->ci;
-}
-
-static inline void qm_dqrr_park(struct qm_portal *portal, u8 idx)
-{
- __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
-
- DPA_ASSERT(dqrr->cmode != qm_dqrr_cdc);
- qm_out(DQRR_DCAP, (0 << 8) | /* S */
- (1 << 6) | /* PK */
- (idx & (QM_DQRR_SIZE - 1))); /* DCAP_CI */
-}
-
-static inline void qm_dqrr_park_current(struct qm_portal *portal)
-{
- register struct qm_dqrr *dqrr = &portal->dqrr;
-
- DPA_ASSERT(dqrr->cmode != qm_dqrr_cdc);
- qm_out(DQRR_DCAP, (0 << 8) | /* S */
- (1 << 6) | /* PK */
- DQRR_PTR2IDX(dqrr->cursor)); /* DCAP_CI */
-}
-
-static inline void qm_dqrr_sdqcr_set(struct qm_portal *portal, u32 sdqcr)
-{
- qm_out(DQRR_SDQCR, sdqcr);
-}
-
-static inline u32 qm_dqrr_sdqcr_get(struct qm_portal *portal)
-{
- return qm_in(DQRR_SDQCR);
-}
-
-static inline void qm_dqrr_vdqcr_set(struct qm_portal *portal, u32 vdqcr)
-{
- qm_out(DQRR_VDQCR, vdqcr);
-}
-
-static inline u32 qm_dqrr_vdqcr_get(struct qm_portal *portal)
-{
- return qm_in(DQRR_VDQCR);
-}
-
-static inline void qm_dqrr_pdqcr_set(struct qm_portal *portal, u32 pdqcr)
-{
- qm_out(DQRR_PDQCR, pdqcr);
-}
-
-static inline u32 qm_dqrr_pdqcr_get(struct qm_portal *portal)
-{
- return qm_in(DQRR_PDQCR);
-}
-
-static inline u8 qm_dqrr_get_ithresh(struct qm_portal *portal)
-{
- register struct qm_dqrr *dqrr = &portal->dqrr;
-
- return dqrr->ithresh;
-}
-
-static inline void qm_dqrr_set_ithresh(struct qm_portal *portal, u8 ithresh)
-{
- qm_out(DQRR_ITR, ithresh);
-}
-
-static inline u8 qm_dqrr_get_maxfill(struct qm_portal *portal)
-{
- return (qm_in(CFG) & 0x00f00000) >> 20;
-}
-
-/* --- MR API --- */
-
-#define MR_CARRYCLEAR(p) \
- (void *)((unsigned long)(p) & (~(unsigned long)(QM_MR_SIZE << 6)))
-
-static inline u8 MR_PTR2IDX(const struct qm_mr_entry *e)
-{
- return ((uintptr_t)e >> 6) & (QM_MR_SIZE - 1);
-}
-
-static inline const struct qm_mr_entry *MR_INC(const struct qm_mr_entry *e)
-{
- return MR_CARRYCLEAR(e + 1);
-}
-
-static inline int qm_mr_init(struct qm_portal *portal, enum qm_mr_pmode pmode,
- enum qm_mr_cmode cmode)
-{
- register struct qm_mr *mr = &portal->mr;
- u32 cfg;
-
- mr->ring = portal->addr.addr_ce + QM_CL_MR;
- mr->pi = qm_in(MR_PI_CINH) & (QM_MR_SIZE - 1);
- mr->ci = qm_in(MR_CI_CINH) & (QM_MR_SIZE - 1);
- mr->cursor = mr->ring + mr->ci;
- mr->fill = qm_cyc_diff(QM_MR_SIZE, mr->ci, mr->pi);
- mr->vbit = (qm_in(MR_PI_CINH) & QM_MR_SIZE) ? QM_MR_VERB_VBIT : 0;
- mr->ithresh = qm_in(MR_ITR);
-#ifdef CONFIG_FSL_DPA_CHECKING
- mr->pmode = pmode;
- mr->cmode = cmode;
-#endif
- cfg = (qm_in(CFG) & 0xfffff0ff) |
- ((cmode & 1) << 8); /* QCSP_CFG:MM */
- qm_out(CFG, cfg);
- return 0;
-}
-
-static inline void qm_mr_finish(struct qm_portal *portal)
-{
- register struct qm_mr *mr = &portal->mr;
-
- if (mr->ci != MR_PTR2IDX(mr->cursor))
- pr_crit("Ignoring completed MR entries\n");
-}
-
-static inline const struct qm_mr_entry *qm_mr_current(struct qm_portal *portal)
-{
- register struct qm_mr *mr = &portal->mr;
-
- if (!mr->fill)
- return NULL;
- return mr->cursor;
-}
-
-static inline u8 qm_mr_cursor(struct qm_portal *portal)
-{
- register struct qm_mr *mr = &portal->mr;
-
- return MR_PTR2IDX(mr->cursor);
-}
-
-static inline u8 qm_mr_next(struct qm_portal *portal)
-{
- register struct qm_mr *mr = &portal->mr;
-
- DPA_ASSERT(mr->fill);
- mr->cursor = MR_INC(mr->cursor);
- return --mr->fill;
-}
-
-static inline u8 qm_mr_pci_update(struct qm_portal *portal)
-{
- register struct qm_mr *mr = &portal->mr;
- u8 diff, old_pi = mr->pi;
-
- DPA_ASSERT(mr->pmode == qm_mr_pci);
- mr->pi = qm_in(MR_PI_CINH);
- diff = qm_cyc_diff(QM_MR_SIZE, old_pi, mr->pi);
- mr->fill += diff;
- return diff;
-}
-
-static inline void qm_mr_pce_prefetch(struct qm_portal *portal)
-{
- __maybe_unused register struct qm_mr *mr = &portal->mr;
-
- DPA_ASSERT(mr->pmode == qm_mr_pce);
- qm_cl_invalidate(MR_PI);
- qm_cl_touch_ro(MR_PI);
-}
-
-static inline u8 qm_mr_pce_update(struct qm_portal *portal)
-{
- register struct qm_mr *mr = &portal->mr;
- u8 diff, old_pi = mr->pi;
-
- DPA_ASSERT(mr->pmode == qm_mr_pce);
- mr->pi = qm_cl_in(MR_PI) & (QM_MR_SIZE - 1);
- diff = qm_cyc_diff(QM_MR_SIZE, old_pi, mr->pi);
- mr->fill += diff;
- return diff;
-}
-
-static inline void qm_mr_pvb_update(struct qm_portal *portal)
-{
- register struct qm_mr *mr = &portal->mr;
- const struct qm_mr_entry *res = qm_cl(mr->ring, mr->pi);
-
- DPA_ASSERT(mr->pmode == qm_mr_pvb);
- /* when accessing 'verb', use __raw_readb() to ensure that compiler
- * inlining doesn't try to optimise out "excess reads". */
- if ((__raw_readb(&res->verb) & QM_MR_VERB_VBIT) == mr->vbit) {
- mr->pi = (mr->pi + 1) & (QM_MR_SIZE - 1);
- if (!mr->pi)
- mr->vbit ^= QM_MR_VERB_VBIT;
- mr->fill++;
- res = MR_INC(res);
- }
- dcbit_ro(res);
-}
-
-static inline void qm_mr_cci_consume(struct qm_portal *portal, u8 num)
-{
- register struct qm_mr *mr = &portal->mr;
-
- DPA_ASSERT(mr->cmode == qm_mr_cci);
- mr->ci = (mr->ci + num) & (QM_MR_SIZE - 1);
- qm_out(MR_CI_CINH, mr->ci);
-}
-
-static inline void qm_mr_cci_consume_to_current(struct qm_portal *portal)
-{
- register struct qm_mr *mr = &portal->mr;
-
- DPA_ASSERT(mr->cmode == qm_mr_cci);
- mr->ci = MR_PTR2IDX(mr->cursor);
- qm_out(MR_CI_CINH, mr->ci);
-}
-
-static inline void qm_mr_cce_prefetch(struct qm_portal *portal)
-{
- __maybe_unused register struct qm_mr *mr = &portal->mr;
-
- DPA_ASSERT(mr->cmode == qm_mr_cce);
- qm_cl_invalidate(MR_CI);
- qm_cl_touch_rw(MR_CI);
-}
-
-static inline void qm_mr_cce_consume(struct qm_portal *portal, u8 num)
-{
- register struct qm_mr *mr = &portal->mr;
-
- DPA_ASSERT(mr->cmode == qm_mr_cce);
- mr->ci = (mr->ci + num) & (QM_MR_SIZE - 1);
- qm_cl_out(MR_CI, mr->ci);
-}
-
-static inline void qm_mr_cce_consume_to_current(struct qm_portal *portal)
-{
- register struct qm_mr *mr = &portal->mr;
-
- DPA_ASSERT(mr->cmode == qm_mr_cce);
- mr->ci = MR_PTR2IDX(mr->cursor);
- qm_cl_out(MR_CI, mr->ci);
-}
-
-static inline u8 qm_mr_get_ci(struct qm_portal *portal)
-{
- register struct qm_mr *mr = &portal->mr;
-
- return mr->ci;
-}
-
-static inline u8 qm_mr_get_ithresh(struct qm_portal *portal)
-{
- register struct qm_mr *mr = &portal->mr;
-
- return mr->ithresh;
-}
-
-static inline void qm_mr_set_ithresh(struct qm_portal *portal, u8 ithresh)
-{
- qm_out(MR_ITR, ithresh);
-}
-
-/* --- Management command API --- */
-
-static inline int qm_mc_init(struct qm_portal *portal)
-{
- register struct qm_mc *mc = &portal->mc;
-
- mc->cr = portal->addr.addr_ce + QM_CL_CR;
- mc->rr = portal->addr.addr_ce + QM_CL_RR0;
- mc->rridx = (__raw_readb(&mc->cr->__dont_write_directly__verb) &
- QM_MCC_VERB_VBIT) ? 0 : 1;
- mc->vbit = mc->rridx ? QM_MCC_VERB_VBIT : 0;
-#ifdef CONFIG_FSL_DPA_CHECKING
- mc->state = qman_mc_idle;
-#endif
- return 0;
-}
-
-static inline void qm_mc_finish(struct qm_portal *portal)
-{
- __maybe_unused register struct qm_mc *mc = &portal->mc;
-
- DPA_ASSERT(mc->state == qman_mc_idle);
-#ifdef CONFIG_FSL_DPA_CHECKING
- if (mc->state != qman_mc_idle)
- pr_crit("Losing incomplete MC command\n");
-#endif
-}
-
-static inline struct qm_mc_command *qm_mc_start(struct qm_portal *portal)
-{
- register struct qm_mc *mc = &portal->mc;
-
- DPA_ASSERT(mc->state == qman_mc_idle);
-#ifdef CONFIG_FSL_DPA_CHECKING
- mc->state = qman_mc_user;
-#endif
- dcbz_64(mc->cr);
- return mc->cr;
-}
-
-static inline void qm_mc_abort(struct qm_portal *portal)
-{
- __maybe_unused register struct qm_mc *mc = &portal->mc;
-
- DPA_ASSERT(mc->state == qman_mc_user);
-#ifdef CONFIG_FSL_DPA_CHECKING
- mc->state = qman_mc_idle;
-#endif
-}
-
-static inline void qm_mc_commit(struct qm_portal *portal, u8 myverb)
-{
- register struct qm_mc *mc = &portal->mc;
- struct qm_mc_result *rr = mc->rr + mc->rridx;
-
- DPA_ASSERT(mc->state == qman_mc_user);
- lwsync();
- mc->cr->__dont_write_directly__verb = myverb | mc->vbit;
- dcbf(mc->cr);
- dcbit_ro(rr);
-#ifdef CONFIG_FSL_DPA_CHECKING
- mc->state = qman_mc_hw;
-#endif
-}
-
-static inline struct qm_mc_result *qm_mc_result(struct qm_portal *portal)
-{
- register struct qm_mc *mc = &portal->mc;
- struct qm_mc_result *rr = mc->rr + mc->rridx;
-
- DPA_ASSERT(mc->state == qman_mc_hw);
- /* The inactive response register's verb byte always returns zero until
- * its command is submitted and completed. This includes the valid-bit,
- * in case you were wondering... */
- if (!__raw_readb(&rr->verb)) {
- dcbit_ro(rr);
- return NULL;
- }
- mc->rridx ^= 1;
- mc->vbit ^= QM_MCC_VERB_VBIT;
-#ifdef CONFIG_FSL_DPA_CHECKING
- mc->state = qman_mc_idle;
-#endif
- return rr;
-}
-
-/* --- Portal interrupt register API --- */
-
-static inline int qm_isr_init(__always_unused struct qm_portal *portal)
-{
- return 0;
-}
-
-static inline void qm_isr_finish(__always_unused struct qm_portal *portal)
-{
-}
-
-static inline void qm_isr_set_iperiod(struct qm_portal *portal, u16 iperiod)
-{
- qm_out(ITPR, iperiod);
-}
-
-static inline u32 __qm_isr_read(struct qm_portal *portal, enum qm_isr_reg n)
-{
- return __qm_in(&portal->addr, QM_REG_ISR + (n << 2));
-}
-
-static inline void __qm_isr_write(struct qm_portal *portal, enum qm_isr_reg n,
- u32 val)
-{
- __qm_out(&portal->addr, QM_REG_ISR + (n << 2), val);
-}