diff options
Diffstat (limited to 'linux/drivers/soc/fsl/qbman/bman_api.c')
| -rw-r--r-- | linux/drivers/soc/fsl/qbman/bman_api.c | 1123 |
1 files changed, 0 insertions, 1123 deletions
diff --git a/linux/drivers/soc/fsl/qbman/bman_api.c b/linux/drivers/soc/fsl/qbman/bman_api.c deleted file mode 100644 index cdfcebbd..00000000 --- a/linux/drivers/soc/fsl/qbman/bman_api.c +++ /dev/null @@ -1,1123 +0,0 @@ -#include <machine/rtems-bsd-kernel-space.h> - -#include <rtems/bsd/local/opt_dpaa.h> - -/* Copyright 2008 - 2015 Freescale Semiconductor, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Freescale Semiconductor nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation, either version 2 of that License or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "bman.h" - -/* Compilation constants */ -#define RCR_THRESH 2 /* reread h/w CI when running out of space */ -#define IRQNAME "BMan portal %d" -#define MAX_IRQNAME 16 /* big enough for "BMan portal %d" */ -#define FSL_DPA_PORTAL_SHARE 1 /* Allow portals to be shared */ - -struct bman_portal { - struct bm_portal p; - /* 2-element array. pools[0] is mask, pools[1] is snapshot. */ - struct bman_depletion *pools; - int thresh_set; - unsigned long irq_sources; - u32 slowpoll; /* only used when interrupts are off */ -#ifdef FSL_DPA_CAN_WAIT_SYNC - struct bman_pool *rcri_owned; /* only 1 release WAIT_SYNC at a time */ -#endif -#ifdef FSL_DPA_PORTAL_SHARE - raw_spinlock_t sharing_lock; /* only used if is_shared */ -#ifndef __rtems__ - int is_shared; - struct bman_portal *sharing_redirect; -#endif /* __rtems__ */ -#endif - /* When the cpu-affine portal is activated, this is non-NULL */ - const struct bm_portal_config *config; - /* 64-entry hash-table of pool objects that are tracking depletion - * entry/exit (ie. BMAN_POOL_FLAG_DEPLETION). This isn't fast-path, so - * we're not fussy about cache-misses and so forth - whereas the above - * members should all fit in one cacheline. - * BTW, with 64 entries in the hash table and 64 buffer pools to track, - * you'll never guess the hash-function ... */ - struct bman_pool *cb[64]; - char irqname[MAX_IRQNAME]; - /* Track if the portal was alloced by the driver */ - u8 alloced; -}; - - -#ifdef FSL_DPA_PORTAL_SHARE -/* For an explanation of the locking, redirection, or affine-portal logic, - * please consult the QMan driver for details. This is the same, only simpler - * (no fiddly QMan-specific bits.) */ -#ifndef __rtems__ -#define PORTAL_IRQ_LOCK(p, irqflags) \ - do { \ - if ((p)->is_shared) \ - raw_spin_lock_irqsave(&(p)->sharing_lock, irqflags); \ - else \ - local_irq_save(irqflags); \ - } while (0) -#define PORTAL_IRQ_UNLOCK(p, irqflags) \ - do { \ - if ((p)->is_shared) \ - raw_spin_unlock_irqrestore(&(p)->sharing_lock, \ - irqflags); \ - else \ - local_irq_restore(irqflags); \ - } while (0) -#else /* __rtems__ */ -#define PORTAL_IRQ_LOCK(p, irqflags) \ - raw_spin_lock_irqsave(&(p)->sharing_lock, irqflags) -#define PORTAL_IRQ_UNLOCK(p, irqflags) \ - raw_spin_unlock_irqrestore(&(p)->sharing_lock, irqflags) -#endif /* __rtems__ */ -#else -#define PORTAL_IRQ_LOCK(p, irqflags) local_irq_save(irqflags) -#define PORTAL_IRQ_UNLOCK(p, irqflags) local_irq_restore(irqflags) -#endif - -#ifndef __rtems__ -static cpumask_t affine_mask; -static DEFINE_SPINLOCK(affine_mask_lock); -#endif /* __rtems__ */ -static DEFINE_PER_CPU(struct bman_portal, bman_affine_portal); -static inline struct bman_portal *get_raw_affine_portal(void) -{ - return &get_cpu_var(bman_affine_portal); -} -#ifdef FSL_DPA_PORTAL_SHARE -static inline struct bman_portal *get_affine_portal(void) -{ - struct bman_portal *p = get_raw_affine_portal(); - -#ifndef __rtems__ - if (p->sharing_redirect) - return p->sharing_redirect; -#endif /* __rtems__ */ - return p; -} -#else -#define get_affine_portal() get_raw_affine_portal() -#endif -static inline void put_affine_portal(void) -{ - put_cpu_var(bman_affine_portal); -} -static inline struct bman_portal *get_poll_portal(void) -{ - return this_cpu_ptr(&bman_affine_portal); -} -#define put_poll_portal() - -/* GOTCHA: this object type refers to a pool, it isn't *the* pool. There may be - * more than one such object per BMan buffer pool, eg. if different users of the - * pool are operating via different portals. */ -struct bman_pool { - struct bman_pool_params params; - /* Used for hash-table admin when using depletion notifications. */ - struct bman_portal *portal; - struct bman_pool *next; - /* stockpile state - NULL unless BMAN_POOL_FLAG_STOCKPILE is set */ - struct bm_buffer *sp; - unsigned int sp_fill; -#ifdef CONFIG_FSL_DPA_CHECKING - atomic_t in_use; -#endif -}; - -/* (De)Registration of depletion notification callbacks */ -static void depletion_link(struct bman_portal *portal, struct bman_pool *pool) -{ - __maybe_unused unsigned long irqflags; - - pool->portal = portal; - PORTAL_IRQ_LOCK(portal, irqflags); - pool->next = portal->cb[pool->params.bpid]; - portal->cb[pool->params.bpid] = pool; - if (!pool->next) - /* First object for that bpid on this portal, enable the BSCN - * mask bit. */ - bm_isr_bscn_mask(&portal->p, pool->params.bpid, 1); - PORTAL_IRQ_UNLOCK(portal, irqflags); -} -static void depletion_unlink(struct bman_pool *pool) -{ - struct bman_pool *it, *last = NULL; - struct bman_pool **base = &pool->portal->cb[pool->params.bpid]; - __maybe_unused unsigned long irqflags; - - PORTAL_IRQ_LOCK(pool->portal, irqflags); - it = *base; /* <-- gotcha, don't do this prior to the irq_save */ - while (it != pool) { - last = it; - it = it->next; - } - if (!last) - *base = pool->next; - else - last->next = pool->next; - if (!last && !pool->next) { - /* Last object for that bpid on this portal, disable the BSCN - * mask bit. */ - bm_isr_bscn_mask(&pool->portal->p, pool->params.bpid, 0); - /* And "forget" that we last saw this pool as depleted */ - bman_depletion_unset(&pool->portal->pools[1], - pool->params.bpid); - } - PORTAL_IRQ_UNLOCK(pool->portal, irqflags); -} - -/* In the case that the application's core loop calls qman_poll() and - * bman_poll(), we ought to balance how often we incur the overheads of the - * slow-path poll. We'll use two decrementer sources. The idle decrementer - * constant is used when the last slow-poll detected no work to do, and the busy - * decrementer constant when the last slow-poll had work to do. */ -#define SLOW_POLL_IDLE 1000 -#define SLOW_POLL_BUSY 10 -static u32 __poll_portal_slow(struct bman_portal *p, u32 is); - -/* Portal interrupt handler */ -static irqreturn_t portal_isr(__always_unused int irq, void *ptr) -{ - struct bman_portal *p = ptr; - u32 clear = p->irq_sources; - u32 is = bm_isr_status_read(&p->p) & p->irq_sources; - - clear |= __poll_portal_slow(p, is); - bm_isr_status_clear(&p->p, clear); - return IRQ_HANDLED; -} - - -struct bman_portal *bman_create_portal( - struct bman_portal *portal, - const struct bm_portal_config *config) -{ - struct bm_portal *__p; - const struct bman_depletion *pools = &config->public_cfg.mask; - int ret; - u8 bpid = 0; - - if (!portal) { - portal = kmalloc(sizeof(*portal), GFP_KERNEL); - if (!portal) - return portal; - portal->alloced = 1; - } else - portal->alloced = 0; - - __p = &portal->p; - - /* prep the low-level portal struct with the mapped addresses from the - * config, everything that follows depends on it and "config" is more - * for (de)reference... */ - __p->addr.addr_ce = config->addr_virt[DPA_PORTAL_CE]; - __p->addr.addr_ci = config->addr_virt[DPA_PORTAL_CI]; - if (bm_rcr_init(__p, bm_rcr_pvb, bm_rcr_cce)) { - pr_err("RCR initialisation failed\n"); - goto fail_rcr; - } - if (bm_mc_init(__p)) { - pr_err("MC initialisation failed\n"); - goto fail_mc; - } - if (bm_isr_init(__p)) { - pr_err("ISR initialisation failed\n"); - goto fail_isr; - } - portal->pools = kmalloc(2 * sizeof(*pools), GFP_KERNEL); - if (!portal->pools) - goto fail_pools; - portal->pools[0] = *pools; - bman_depletion_init(portal->pools + 1); - while (bpid < bman_pool_max) { - /* Default to all BPIDs disabled, we enable as required at - * run-time. */ - bm_isr_bscn_mask(__p, bpid, 0); - bpid++; - } - portal->slowpoll = 0; -#ifdef FSL_DPA_CAN_WAIT_SYNC - portal->rcri_owned = NULL; -#endif -#ifdef FSL_DPA_PORTAL_SHARE - raw_spin_lock_init(&portal->sharing_lock); -#ifndef __rtems__ - portal->is_shared = config->public_cfg.is_shared; - portal->sharing_redirect = NULL; -#endif /* __rtems__ */ -#endif - memset(&portal->cb, 0, sizeof(portal->cb)); - /* Write-to-clear any stale interrupt status bits */ - bm_isr_disable_write(__p, 0xffffffff); - portal->irq_sources = 0; - bm_isr_enable_write(__p, portal->irq_sources); - bm_isr_status_clear(__p, 0xffffffff); - snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, config->public_cfg.cpu); - if (request_irq(config->public_cfg.irq, portal_isr, 0, portal->irqname, - portal)) { - pr_err("request_irq() failed\n"); - goto fail_irq; - } -#ifndef __rtems__ - if ((config->public_cfg.cpu != -1) && - irq_can_set_affinity(config->public_cfg.irq) && - irq_set_affinity(config->public_cfg.irq, - cpumask_of(config->public_cfg.cpu))) { - pr_err("irq_set_affinity() failed\n"); - goto fail_affinity; - } -#endif /* __rtems__ */ - - /* Need RCR to be empty before continuing */ - ret = bm_rcr_get_fill(__p); - if (ret) { - pr_err("RCR unclean\n"); - goto fail_rcr_empty; - } - /* Success */ - portal->config = config; - - bm_isr_disable_write(__p, 0); - bm_isr_uninhibit(__p); - return portal; -fail_rcr_empty: -#ifndef __rtems__ -fail_affinity: -#endif /* __rtems__ */ - free_irq(config->public_cfg.irq, portal); -fail_irq: - kfree(portal->pools); -fail_pools: - bm_isr_finish(__p); -fail_isr: - bm_mc_finish(__p); -fail_mc: - bm_rcr_finish(__p); -fail_rcr: - if (portal->alloced) - kfree(portal); - return NULL; -} - -struct bman_portal *bman_create_affine_portal( - const struct bm_portal_config *config) -{ - struct bman_portal *portal; - - portal = &per_cpu(bman_affine_portal, config->public_cfg.cpu); - portal = bman_create_portal(portal, config); -#ifndef __rtems__ - if (portal) { - spin_lock(&affine_mask_lock); - cpumask_set_cpu(config->public_cfg.cpu, &affine_mask); - spin_unlock(&affine_mask_lock); - } -#endif /* __rtems__ */ - return portal; -} - - -#ifndef __rtems__ -struct bman_portal *bman_create_affine_slave(struct bman_portal *redirect, - int cpu) -{ -#ifdef FSL_DPA_PORTAL_SHARE - struct bman_portal *p = &per_cpu(bman_affine_portal, cpu); - - BUG_ON(p->config); - BUG_ON(p->is_shared); - BUG_ON(!redirect->config->public_cfg.is_shared); - p->irq_sources = 0; - p->sharing_redirect = redirect; - put_affine_portal(); - return p; -#else - BUG(); - return NULL; -#endif -} -#endif /* __rtems__ */ - -void bman_destroy_portal(struct bman_portal *bm) -{ - const struct bm_portal_config *pcfg = bm->config; - - bm_rcr_cce_update(&bm->p); - bm_rcr_cce_update(&bm->p); - - free_irq(pcfg->public_cfg.irq, bm); - - kfree(bm->pools); - bm_isr_finish(&bm->p); - bm_mc_finish(&bm->p); - bm_rcr_finish(&bm->p); - bm->config = NULL; - if (bm->alloced) - kfree(bm); -} - -const struct bm_portal_config *bman_destroy_affine_portal(void) -{ - struct bman_portal *bm = get_raw_affine_portal(); - const struct bm_portal_config *pcfg; - -#ifdef FSL_DPA_PORTAL_SHARE -#ifndef __rtems__ - if (bm->sharing_redirect) { - bm->sharing_redirect = NULL; - put_affine_portal(); - return NULL; - } - bm->is_shared = 0; -#endif /* __rtems__ */ -#endif - pcfg = bm->config; - bman_destroy_portal(bm); -#ifndef __rtems__ - spin_lock(&affine_mask_lock); - cpumask_clear_cpu(pcfg->public_cfg.cpu, &affine_mask); - spin_unlock(&affine_mask_lock); -#endif /* __rtems__ */ - put_affine_portal(); - return pcfg; -} - -/* When release logic waits on available RCR space, we need a global waitqueue - * in the case of "affine" use (as the waits wake on different cpus which means - * different portals - so we can't wait on any per-portal waitqueue). */ -static DECLARE_WAIT_QUEUE_HEAD(affine_queue); - -static u32 __poll_portal_slow(struct bman_portal *p, u32 is) -{ - struct bman_depletion tmp; - u32 ret = is; - - /* There is a gotcha to be aware of. If we do the query before clearing - * the status register, we may miss state changes that occur between the - * two. If we write to clear the status register before the query, the - * cache-enabled query command may overtake the status register write - * unless we use a heavyweight sync (which we don't want). Instead, we - * write-to-clear the status register then *read it back* before doing - * the query, hence the odd while loop with the 'is' accumulation. */ - if (is & BM_PIRQ_BSCN) { - struct bm_mc_result *mcr; - __maybe_unused unsigned long irqflags; - unsigned int i, j; - u32 __is; - - bm_isr_status_clear(&p->p, BM_PIRQ_BSCN); - while ((__is = bm_isr_status_read(&p->p)) & BM_PIRQ_BSCN) { - is |= __is; - bm_isr_status_clear(&p->p, BM_PIRQ_BSCN); - } - is &= ~BM_PIRQ_BSCN; - PORTAL_IRQ_LOCK(p, irqflags); - bm_mc_start(&p->p); - bm_mc_commit(&p->p, BM_MCC_VERB_CMD_QUERY); - while (!(mcr = bm_mc_result(&p->p))) - cpu_relax(); - tmp = mcr->query.ds.state; - PORTAL_IRQ_UNLOCK(p, irqflags); - for (i = 0; i < 2; i++) { - int idx = i * 32; - /* tmp is a mask of currently-depleted pools. - * pools[0] is mask of those we care about. - * pools[1] is our previous view (we only want to - * be told about changes). */ - tmp.__state[i] &= p->pools[0].__state[i]; - if (tmp.__state[i] == p->pools[1].__state[i]) - /* fast-path, nothing to see, move along */ - continue; - for (j = 0; j <= 31; j++, idx++) { - struct bman_pool *pool = p->cb[idx]; - int b4 = bman_depletion_get(&p->pools[1], idx); - int af = bman_depletion_get(&tmp, idx); - - if (b4 == af) - continue; - while (pool) { - pool->params.cb(p, pool, - pool->params.cb_ctx, af); - pool = pool->next; - } - } - } - p->pools[1] = tmp; - } - - if (is & BM_PIRQ_RCRI) { - __maybe_unused unsigned long irqflags; - - PORTAL_IRQ_LOCK(p, irqflags); - bm_rcr_cce_update(&p->p); -#ifdef FSL_DPA_CAN_WAIT_SYNC - /* If waiting for sync, we only cancel the interrupt threshold - * when the ring utilisation hits zero. */ - if (p->rcri_owned) { - if (!bm_rcr_get_fill(&p->p)) { - p->rcri_owned = NULL; - bm_rcr_set_ithresh(&p->p, 0); - } - } else -#endif - bm_rcr_set_ithresh(&p->p, 0); - PORTAL_IRQ_UNLOCK(p, irqflags); - wake_up(&affine_queue); - bm_isr_status_clear(&p->p, BM_PIRQ_RCRI); - is &= ~BM_PIRQ_RCRI; - } - - /* There should be no status register bits left undefined */ - DPA_ASSERT(!is); - return ret; -} - -const struct bman_portal_config *bman_get_portal_config(void) -{ - struct bman_portal *p = get_affine_portal(); - const struct bman_portal_config *ret = &p->config->public_cfg; - - put_affine_portal(); - return ret; -} -EXPORT_SYMBOL(bman_get_portal_config); - -u32 bman_irqsource_get(void) -{ - struct bman_portal *p = get_raw_affine_portal(); - u32 ret = p->irq_sources & BM_PIRQ_VISIBLE; - - put_affine_portal(); - return ret; -} -EXPORT_SYMBOL(bman_irqsource_get); - -int bman_p_irqsource_add(struct bman_portal *p, __maybe_unused u32 bits) -{ - __maybe_unused unsigned long irqflags; - -#ifdef FSL_DPA_PORTAL_SHARE -#ifndef __rtems__ - if (p->sharing_redirect) - return -EINVAL; -#endif /* __rtems__ */ -#endif - PORTAL_IRQ_LOCK(p, irqflags); - set_bits(bits & BM_PIRQ_VISIBLE, &p->irq_sources); - bm_isr_enable_write(&p->p, p->irq_sources); - PORTAL_IRQ_UNLOCK(p, irqflags); - return 0; -} -EXPORT_SYMBOL(bman_p_irqsource_add); - -int bman_irqsource_add(__maybe_unused u32 bits) -{ - struct bman_portal *p = get_raw_affine_portal(); - int ret = bman_p_irqsource_add(p, bits); - - put_affine_portal(); - return ret; -} -EXPORT_SYMBOL(bman_irqsource_add); - -int bman_irqsource_remove(u32 bits) -{ - struct bman_portal *p = get_raw_affine_portal(); - __maybe_unused unsigned long irqflags; - u32 ier; - -#ifdef FSL_DPA_PORTAL_SHARE -#ifndef __rtems__ - if (p->sharing_redirect) { - put_affine_portal(); - return -EINVAL; - } -#endif /* __rtems__ */ -#endif - /* Our interrupt handler only processes+clears status register bits that - * are in p->irq_sources. As we're trimming that mask, if one of them - * were to assert in the status register just before we remove it from - * the enable register, there would be an interrupt-storm when we - * release the IRQ lock. So we wait for the enable register update to - * take effect in h/w (by reading it back) and then clear all other bits - * in the status register. Ie. we clear them from ISR once it's certain - * IER won't allow them to reassert. */ - PORTAL_IRQ_LOCK(p, irqflags); - bits &= BM_PIRQ_VISIBLE; - clear_bits(bits, &p->irq_sources); - bm_isr_enable_write(&p->p, p->irq_sources); - ier = bm_isr_enable_read(&p->p); - /* Using "~ier" (rather than "bits" or "~p->irq_sources") creates a - * data-dependency, ie. to protect against re-ordering. */ - bm_isr_status_clear(&p->p, ~ier); - PORTAL_IRQ_UNLOCK(p, irqflags); - put_affine_portal(); - return 0; -} -EXPORT_SYMBOL(bman_irqsource_remove); - -#ifndef __rtems__ -const cpumask_t *bman_affine_cpus(void) -{ - return &affine_mask; -} -EXPORT_SYMBOL(bman_affine_cpus); -#endif /* __rtems__ */ - -u32 bman_poll_slow(void) -{ - struct bman_portal *p = get_poll_portal(); - u32 ret; - -#ifdef FSL_DPA_PORTAL_SHARE -#ifndef __rtems__ - if (unlikely(p->sharing_redirect)) - ret = (u32)-1; - else -#endif /* __rtems__ */ -#endif - { - u32 is = bm_isr_status_read(&p->p) & ~p->irq_sources; - - ret = __poll_portal_slow(p, is); - bm_isr_status_clear(&p->p, ret); - } - put_poll_portal(); - return ret; -} -EXPORT_SYMBOL(bman_poll_slow); - -/* Legacy wrapper */ -void bman_poll(void) -{ - struct bman_portal *p = get_poll_portal(); - -#ifdef FSL_DPA_PORTAL_SHARE -#ifndef __rtems__ - if (unlikely(p->sharing_redirect)) - goto done; -#endif /* __rtems__ */ -#endif - if (!(p->slowpoll--)) { - u32 is = bm_isr_status_read(&p->p) & ~p->irq_sources; - u32 active = __poll_portal_slow(p, is); - - if (active) - p->slowpoll = SLOW_POLL_BUSY; - else - p->slowpoll = SLOW_POLL_IDLE; - } -#ifdef FSL_DPA_PORTAL_SHARE -#ifndef __rtems__ -done: -#endif /* __rtems__ */ -#endif - put_poll_portal(); -} -EXPORT_SYMBOL(bman_poll); - -static const u32 zero_thresholds[4] = {0, 0, 0, 0}; - -struct bman_pool *bman_new_pool(const struct bman_pool_params *params) -{ - struct bman_pool *pool = NULL; - u32 bpid; - - if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID) { - if (bman_alloc_bpid(&bpid)) - return NULL; - } else { - if (params->bpid >= bman_pool_max) - return NULL; - bpid = params->bpid; - } -#ifdef CONFIG_FSL_BMAN - if (params->flags & BMAN_POOL_FLAG_THRESH) { - if (bm_pool_set(bpid, params->thresholds)) - goto err; - } -#else - if (params->flags & BMAN_POOL_FLAG_THRESH) - goto err; -#endif - pool = kmalloc(sizeof(*pool), GFP_KERNEL); - if (!pool) - goto err; - pool->sp = NULL; - pool->sp_fill = 0; - pool->params = *params; -#ifdef CONFIG_FSL_DPA_CHECKING - atomic_set(&pool->in_use, 1); -#endif - if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID) - pool->params.bpid = bpid; - if (params->flags & BMAN_POOL_FLAG_STOCKPILE) { - pool->sp = kmalloc(sizeof(struct bm_buffer) * BMAN_STOCKPILE_SZ, - GFP_KERNEL); - if (!pool->sp) - goto err; - } - if (pool->params.flags & BMAN_POOL_FLAG_DEPLETION) { - struct bman_portal *p = get_affine_portal(); - - if (!p->pools || !bman_depletion_get(&p->pools[0], bpid)) { - pr_err("Depletion events disabled for bpid %d\n", bpid); - goto err; - } - depletion_link(p, pool); - put_affine_portal(); - } - return pool; -err: -#ifdef CONFIG_FSL_BMAN - if (params->flags & BMAN_POOL_FLAG_THRESH) - bm_pool_set(bpid, zero_thresholds); -#endif - if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID) - bman_release_bpid(bpid); - if (pool) { - kfree(pool->sp); - kfree(pool); - } - return NULL; -} -EXPORT_SYMBOL(bman_new_pool); - -void bman_free_pool(struct bman_pool *pool) -{ -#ifdef CONFIG_FSL_BMAN - if (pool->params.flags & BMAN_POOL_FLAG_THRESH) - bm_pool_set(pool->params.bpid, zero_thresholds); -#endif - if (pool->params.flags & BMAN_POOL_FLAG_DEPLETION) - depletion_unlink(pool); - if (pool->params.flags & BMAN_POOL_FLAG_STOCKPILE) { - if (pool->sp_fill) - pr_err("Stockpile not flushed, has %u in bpid %u.\n", - pool->sp_fill, pool->params.bpid); - kfree(pool->sp); - pool->sp = NULL; - pool->params.flags ^= BMAN_POOL_FLAG_STOCKPILE; - } - if (pool->params.flags & BMAN_POOL_FLAG_DYNAMIC_BPID) - bman_release_bpid(pool->params.bpid); - kfree(pool); -} -EXPORT_SYMBOL(bman_free_pool); - -const struct bman_pool_params *bman_get_params(const struct bman_pool *pool) -{ - return &pool->params; -} -EXPORT_SYMBOL(bman_get_params); - -static noinline void update_rcr_ci(struct bman_portal *p, u8 avail) -{ - if (avail) - bm_rcr_cce_prefetch(&p->p); - else - bm_rcr_cce_update(&p->p); -} - -int bman_rcr_is_empty(void) -{ - __maybe_unused unsigned long irqflags; - struct bman_portal *p = get_affine_portal(); - u8 avail; - - PORTAL_IRQ_LOCK(p, irqflags); - update_rcr_ci(p, 0); - avail = bm_rcr_get_fill(&p->p); - PORTAL_IRQ_UNLOCK(p, irqflags); - put_affine_portal(); - return avail == 0; -} -EXPORT_SYMBOL(bman_rcr_is_empty); - -static inline struct bm_rcr_entry *try_rel_start(struct bman_portal **p, -#ifdef FSL_DPA_CAN_WAIT - __maybe_unused struct bman_pool *pool, -#endif - __maybe_unused unsigned long *irqflags, - __maybe_unused u32 flags) -{ - struct bm_rcr_entry *r; - u8 avail; - - *p = get_affine_portal(); - PORTAL_IRQ_LOCK(*p, (*irqflags)); -#ifdef FSL_DPA_CAN_WAIT_SYNC - if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) && - (flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) { - if ((*p)->rcri_owned) { - PORTAL_IRQ_UNLOCK(*p, (*irqflags)); - put_affine_portal(); - return NULL; - } - (*p)->rcri_owned = pool; - } -#endif - avail = bm_rcr_get_avail(&(*p)->p); - if (avail < 2) - update_rcr_ci(*p, avail); - r = bm_rcr_start(&(*p)->p); - if (unlikely(!r)) { -#ifdef FSL_DPA_CAN_WAIT_SYNC - if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) && - (flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) - (*p)->rcri_owned = NULL; -#endif - PORTAL_IRQ_UNLOCK(*p, (*irqflags)); - put_affine_portal(); - } - return r; -} - -#ifdef FSL_DPA_CAN_WAIT -static noinline struct bm_rcr_entry *__wait_rel_start(struct bman_portal **p, - struct bman_pool *pool, - __maybe_unused unsigned long *irqflags, - u32 flags) -{ - struct bm_rcr_entry *rcr = try_rel_start(p, pool, irqflags, flags); - - if (!rcr) - bm_rcr_set_ithresh(&(*p)->p, 1); - return rcr; -} - -static noinline struct bm_rcr_entry *wait_rel_start(struct bman_portal **p, - struct bman_pool *pool, - __maybe_unused unsigned long *irqflags, - u32 flags) -{ - struct bm_rcr_entry *rcr; -#ifndef FSL_DPA_CAN_WAIT_SYNC - pool = NULL; -#endif -#ifndef __rtems__ - if (flags & BMAN_RELEASE_FLAG_WAIT_INT) - wait_event_interruptible(affine_queue, - (rcr = __wait_rel_start(p, pool, irqflags, flags))); - else -#endif /* __rtems__ */ - wait_event(affine_queue, - (rcr = __wait_rel_start(p, pool, irqflags, flags))); - return rcr; -} -#endif - -/* to facilitate better copying of bufs into the ring without either (a) copying - * noise into the first byte (prematurely triggering the command), nor (b) being - * very inefficient by copying small fields using read-modify-write */ -struct overlay_bm_buffer { - u32 first; - u32 second; -}; - -static inline int __bman_release(struct bman_pool *pool, - const struct bm_buffer *bufs, u8 num, u32 flags) -{ - struct bman_portal *p; - struct bm_rcr_entry *r; - struct overlay_bm_buffer *o_dest; - struct overlay_bm_buffer *o_src = (struct overlay_bm_buffer *)&bufs[0]; - __maybe_unused unsigned long irqflags; - u32 i = num - 1; - -#ifdef FSL_DPA_CAN_WAIT - if (flags & BMAN_RELEASE_FLAG_WAIT) - r = wait_rel_start(&p, pool, &irqflags, flags); - else - r = try_rel_start(&p, pool, &irqflags, flags); -#else - r = try_rel_start(&p, &irqflags, flags); -#endif - if (!r) - return -EBUSY; - /* We can copy all but the first entry, as this can trigger badness - * with the valid-bit. Use the overlay to mask the verb byte. */ - o_dest = (struct overlay_bm_buffer *)&r->bufs[0]; - o_dest->first = (o_src->first & 0x0000ffff) | - (((u32)pool->params.bpid << 16) & 0x00ff0000); - o_dest->second = o_src->second; - if (i) - copy_words(&r->bufs[1], &bufs[1], i * sizeof(bufs[0])); - bm_rcr_pvb_commit(&p->p, BM_RCR_VERB_CMD_BPID_SINGLE | - (num & BM_RCR_VERB_BUFCOUNT_MASK)); -#ifdef FSL_DPA_CAN_WAIT_SYNC - /* if we wish to sync we need to set the threshold after h/w sees the - * new ring entry. As we're mixing cache-enabled and cache-inhibited - * accesses, this requires a heavy-weight sync. */ - if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) && - (flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) { - hwsync(); - bm_rcr_set_ithresh(&p->p, 1); - } -#endif - PORTAL_IRQ_UNLOCK(p, irqflags); - put_affine_portal(); -#ifdef FSL_DPA_CAN_WAIT_SYNC - if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) && - (flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) { -#ifndef __rtems__ - if (flags & BMAN_RELEASE_FLAG_WAIT_INT) - wait_event_interruptible(affine_queue, - (p->rcri_owned != pool)); - else -#endif /* __rtems__ */ - wait_event(affine_queue, (p->rcri_owned != pool)); - } -#endif - return 0; -} - -int bman_release(struct bman_pool *pool, const struct bm_buffer *bufs, u8 num, - u32 flags) -{ - int ret = 0; - -#ifdef CONFIG_FSL_DPA_CHECKING - if (!num || (num > 8)) - return -EINVAL; - if (pool->params.flags & BMAN_POOL_FLAG_NO_RELEASE) - return -EINVAL; -#endif - /* Without stockpile, this API is a pass-through to the h/w operation */ - if (!(pool->params.flags & BMAN_POOL_FLAG_STOCKPILE)) - return __bman_release(pool, bufs, num, flags); -#ifdef CONFIG_FSL_DPA_CHECKING - if (!atomic_dec_and_test(&pool->in_use)) { - pr_crit("Parallel attempts to enter bman_released() detected."); - panic("only one instance of bman_released/acquired allowed"); - } -#endif - /* This needs some explanation. Adding the given buffers may take the - * stockpile over the threshold, but in fact the stockpile may already - * *be* over the threshold if a previous release-to-hw attempt had - * failed. So we have 3 cases to cover; - * 1. we add to the stockpile and don't hit the threshold, - * 2. we add to the stockpile, hit the threshold and release-to-hw, - * 3. we have to release-to-hw before adding to the stockpile - * (not enough room in the stockpile for case 2). - * Our constraints on thresholds guarantee that in case 3, there must be - * at least 8 bufs already in the stockpile, so all release-to-hw ops - * are for 8 bufs. Despite all this, the API must indicate whether the - * given buffers were taken off the caller's hands, irrespective of - * whether a release-to-hw was attempted. */ - while (num) { - /* Add buffers to stockpile if they fit */ - if ((pool->sp_fill + num) < BMAN_STOCKPILE_SZ) { - copy_words(pool->sp + pool->sp_fill, bufs, - sizeof(struct bm_buffer) * num); - pool->sp_fill += num; - num = 0; /* --> will return success no matter what */ - } - /* Do hw op if hitting the high-water threshold */ - if ((pool->sp_fill + num) >= BMAN_STOCKPILE_HIGH) { - ret = __bman_release(pool, - pool->sp + (pool->sp_fill - 8), 8, flags); - if (ret) { - ret = (num ? ret : 0); - goto release_done; - } - pool->sp_fill -= 8; - } - } -release_done: -#ifdef CONFIG_FSL_DPA_CHECKING - atomic_inc(&pool->in_use); -#endif - return ret; -} -EXPORT_SYMBOL(bman_release); - -static inline int __bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, - u8 num) -{ - struct bman_portal *p = get_affine_portal(); - struct bm_mc_command *mcc; - struct bm_mc_result *mcr; - __maybe_unused unsigned long irqflags; - int ret; - - PORTAL_IRQ_LOCK(p, irqflags); - mcc = bm_mc_start(&p->p); - mcc->acquire.bpid = pool->params.bpid; - bm_mc_commit(&p->p, BM_MCC_VERB_CMD_ACQUIRE | - (num & BM_MCC_VERB_ACQUIRE_BUFCOUNT)); - while (!(mcr = bm_mc_result(&p->p))) - cpu_relax(); - ret = mcr->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT; - if (bufs) - copy_words(&bufs[0], &mcr->acquire.bufs[0], - num * sizeof(bufs[0])); - PORTAL_IRQ_UNLOCK(p, irqflags); - put_affine_portal(); - if (ret != num) - ret = -ENOMEM; - return ret; -} - -int bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, u8 num, - u32 flags) -{ - int ret = 0; - -#ifdef CONFIG_FSL_DPA_CHECKING - if (!num || (num > 8)) - return -EINVAL; - if (pool->params.flags & BMAN_POOL_FLAG_ONLY_RELEASE) - return -EINVAL; -#endif - /* Without stockpile, this API is a pass-through to the h/w operation */ - if (!(pool->params.flags & BMAN_POOL_FLAG_STOCKPILE)) - return __bman_acquire(pool, bufs, num); -#ifdef CONFIG_FSL_DPA_CHECKING - if (!atomic_dec_and_test(&pool->in_use)) { - pr_crit("Parallel attempts to enter bman_acquire() detected."); - panic("only one instance of bman_released/acquired allowed"); - } -#endif - /* Only need a h/w op if we'll hit the low-water thresh */ - if (!(flags & BMAN_ACQUIRE_FLAG_STOCKPILE) && - (pool->sp_fill <= (BMAN_STOCKPILE_LOW + num))) { - /* refill stockpile with max amount, but if max amount - * isn't available, try amount the user wants */ - int bufcount = 8; - - ret = __bman_acquire(pool, pool->sp + pool->sp_fill, bufcount); - if (ret < 0 && bufcount != num) { - bufcount = num; - /* Maybe buffer pool has less than 8 */ - ret = __bman_acquire(pool, pool->sp + pool->sp_fill, - bufcount); - } - if (ret < 0) - goto hw_starved; - DPA_ASSERT(ret == bufcount); - pool->sp_fill += bufcount; - } else { -hw_starved: - if (pool->sp_fill < num) { - ret = -ENOMEM; - goto acquire_done; - } - } - copy_words(bufs, pool->sp + (pool->sp_fill - num), - sizeof(struct bm_buffer) * num); - pool->sp_fill -= num; - ret = num; -acquire_done: -#ifdef CONFIG_FSL_DPA_CHECKING - atomic_inc(&pool->in_use); -#endif - return ret; -} -EXPORT_SYMBOL(bman_acquire); - -int bman_flush_stockpile(struct bman_pool *pool, u32 flags) -{ - u8 num; - int ret; - - while (pool->sp_fill) { - num = ((pool->sp_fill > 8) ? 8 : pool->sp_fill); - ret = __bman_release(pool, pool->sp + (pool->sp_fill - num), - num, flags); - if (ret) - return ret; - pool->sp_fill -= num; - } - return 0; -} -EXPORT_SYMBOL(bman_flush_stockpile); - -int bman_query_pools(struct bm_pool_state *state) -{ - struct bman_portal *p = get_affine_portal(); - struct bm_mc_result *mcr; - __maybe_unused unsigned long irqflags; - - PORTAL_IRQ_LOCK(p, irqflags); - bm_mc_start(&p->p); - bm_mc_commit(&p->p, BM_MCC_VERB_CMD_QUERY); - while (!(mcr = bm_mc_result(&p->p))) - cpu_relax(); - DPA_ASSERT((mcr->verb & BM_MCR_VERB_CMD_MASK) == BM_MCR_VERB_CMD_QUERY); - *state = mcr->query; - PORTAL_IRQ_UNLOCK(p, irqflags); - put_affine_portal(); - return 0; -} -EXPORT_SYMBOL(bman_query_pools); - -#ifdef CONFIG_FSL_BMAN -u32 bman_query_free_buffers(struct bman_pool *pool) -{ - return bm_pool_free_buffers(pool->params.bpid); -} -EXPORT_SYMBOL(bman_query_free_buffers); - -int bman_update_pool_thresholds(struct bman_pool *pool, const u32 *thresholds) -{ - u32 bpid; - - bpid = bman_get_params(pool)->bpid; - - return bm_pool_set(bpid, thresholds); -} -EXPORT_SYMBOL(bman_update_pool_thresholds); -#endif - -int bman_shutdown_pool(u32 bpid) -{ - struct bman_portal *p = get_affine_portal(); - __maybe_unused unsigned long irqflags; - int ret; - - PORTAL_IRQ_LOCK(p, irqflags); - ret = bm_shutdown_pool(&p->p, bpid); - PORTAL_IRQ_UNLOCK(p, irqflags); - put_affine_portal(); - return ret; -} -EXPORT_SYMBOL(bman_shutdown_pool); - -const struct bm_portal_config * -bman_get_bm_portal_config(const struct bman_portal *portal) -{ -#ifndef __rtems__ - return portal->sharing_redirect ? NULL : portal->config; -#else /* __rtems__ */ - return portal->config; -#endif /* __rtems__ */ -} |