diff options
Diffstat (limited to 'linux/drivers/soc/fsl/qbman/bman_api.c')
| -rw-r--r-- | linux/drivers/soc/fsl/qbman/bman_api.c | 1123 |
1 files changed, 1123 insertions, 0 deletions
diff --git a/linux/drivers/soc/fsl/qbman/bman_api.c b/linux/drivers/soc/fsl/qbman/bman_api.c new file mode 100644 index 00000000..cdfcebbd --- /dev/null +++ b/linux/drivers/soc/fsl/qbman/bman_api.c @@ -0,0 +1,1123 @@ +#include <machine/rtems-bsd-kernel-space.h> + +#include <rtems/bsd/local/opt_dpaa.h> + +/* Copyright 2008 - 2015 Freescale Semiconductor, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Freescale Semiconductor nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "bman.h" + +/* Compilation constants */ +#define RCR_THRESH 2 /* reread h/w CI when running out of space */ +#define IRQNAME "BMan portal %d" +#define MAX_IRQNAME 16 /* big enough for "BMan portal %d" */ +#define FSL_DPA_PORTAL_SHARE 1 /* Allow portals to be shared */ + +struct bman_portal { + struct bm_portal p; + /* 2-element array. pools[0] is mask, pools[1] is snapshot. */ + struct bman_depletion *pools; + int thresh_set; + unsigned long irq_sources; + u32 slowpoll; /* only used when interrupts are off */ +#ifdef FSL_DPA_CAN_WAIT_SYNC + struct bman_pool *rcri_owned; /* only 1 release WAIT_SYNC at a time */ +#endif +#ifdef FSL_DPA_PORTAL_SHARE + raw_spinlock_t sharing_lock; /* only used if is_shared */ +#ifndef __rtems__ + int is_shared; + struct bman_portal *sharing_redirect; +#endif /* __rtems__ */ +#endif + /* When the cpu-affine portal is activated, this is non-NULL */ + const struct bm_portal_config *config; + /* 64-entry hash-table of pool objects that are tracking depletion + * entry/exit (ie. BMAN_POOL_FLAG_DEPLETION). This isn't fast-path, so + * we're not fussy about cache-misses and so forth - whereas the above + * members should all fit in one cacheline. + * BTW, with 64 entries in the hash table and 64 buffer pools to track, + * you'll never guess the hash-function ... */ + struct bman_pool *cb[64]; + char irqname[MAX_IRQNAME]; + /* Track if the portal was alloced by the driver */ + u8 alloced; +}; + + +#ifdef FSL_DPA_PORTAL_SHARE +/* For an explanation of the locking, redirection, or affine-portal logic, + * please consult the QMan driver for details. This is the same, only simpler + * (no fiddly QMan-specific bits.) */ +#ifndef __rtems__ +#define PORTAL_IRQ_LOCK(p, irqflags) \ + do { \ + if ((p)->is_shared) \ + raw_spin_lock_irqsave(&(p)->sharing_lock, irqflags); \ + else \ + local_irq_save(irqflags); \ + } while (0) +#define PORTAL_IRQ_UNLOCK(p, irqflags) \ + do { \ + if ((p)->is_shared) \ + raw_spin_unlock_irqrestore(&(p)->sharing_lock, \ + irqflags); \ + else \ + local_irq_restore(irqflags); \ + } while (0) +#else /* __rtems__ */ +#define PORTAL_IRQ_LOCK(p, irqflags) \ + raw_spin_lock_irqsave(&(p)->sharing_lock, irqflags) +#define PORTAL_IRQ_UNLOCK(p, irqflags) \ + raw_spin_unlock_irqrestore(&(p)->sharing_lock, irqflags) +#endif /* __rtems__ */ +#else +#define PORTAL_IRQ_LOCK(p, irqflags) local_irq_save(irqflags) +#define PORTAL_IRQ_UNLOCK(p, irqflags) local_irq_restore(irqflags) +#endif + +#ifndef __rtems__ +static cpumask_t affine_mask; +static DEFINE_SPINLOCK(affine_mask_lock); +#endif /* __rtems__ */ +static DEFINE_PER_CPU(struct bman_portal, bman_affine_portal); +static inline struct bman_portal *get_raw_affine_portal(void) +{ + return &get_cpu_var(bman_affine_portal); +} +#ifdef FSL_DPA_PORTAL_SHARE +static inline struct bman_portal *get_affine_portal(void) +{ + struct bman_portal *p = get_raw_affine_portal(); + +#ifndef __rtems__ + if (p->sharing_redirect) + return p->sharing_redirect; +#endif /* __rtems__ */ + return p; +} +#else +#define get_affine_portal() get_raw_affine_portal() +#endif +static inline void put_affine_portal(void) +{ + put_cpu_var(bman_affine_portal); +} +static inline struct bman_portal *get_poll_portal(void) +{ + return this_cpu_ptr(&bman_affine_portal); +} +#define put_poll_portal() + +/* GOTCHA: this object type refers to a pool, it isn't *the* pool. There may be + * more than one such object per BMan buffer pool, eg. if different users of the + * pool are operating via different portals. */ +struct bman_pool { + struct bman_pool_params params; + /* Used for hash-table admin when using depletion notifications. */ + struct bman_portal *portal; + struct bman_pool *next; + /* stockpile state - NULL unless BMAN_POOL_FLAG_STOCKPILE is set */ + struct bm_buffer *sp; + unsigned int sp_fill; +#ifdef CONFIG_FSL_DPA_CHECKING + atomic_t in_use; +#endif +}; + +/* (De)Registration of depletion notification callbacks */ +static void depletion_link(struct bman_portal *portal, struct bman_pool *pool) +{ + __maybe_unused unsigned long irqflags; + + pool->portal = portal; + PORTAL_IRQ_LOCK(portal, irqflags); + pool->next = portal->cb[pool->params.bpid]; + portal->cb[pool->params.bpid] = pool; + if (!pool->next) + /* First object for that bpid on this portal, enable the BSCN + * mask bit. */ + bm_isr_bscn_mask(&portal->p, pool->params.bpid, 1); + PORTAL_IRQ_UNLOCK(portal, irqflags); +} +static void depletion_unlink(struct bman_pool *pool) +{ + struct bman_pool *it, *last = NULL; + struct bman_pool **base = &pool->portal->cb[pool->params.bpid]; + __maybe_unused unsigned long irqflags; + + PORTAL_IRQ_LOCK(pool->portal, irqflags); + it = *base; /* <-- gotcha, don't do this prior to the irq_save */ + while (it != pool) { + last = it; + it = it->next; + } + if (!last) + *base = pool->next; + else + last->next = pool->next; + if (!last && !pool->next) { + /* Last object for that bpid on this portal, disable the BSCN + * mask bit. */ + bm_isr_bscn_mask(&pool->portal->p, pool->params.bpid, 0); + /* And "forget" that we last saw this pool as depleted */ + bman_depletion_unset(&pool->portal->pools[1], + pool->params.bpid); + } + PORTAL_IRQ_UNLOCK(pool->portal, irqflags); +} + +/* In the case that the application's core loop calls qman_poll() and + * bman_poll(), we ought to balance how often we incur the overheads of the + * slow-path poll. We'll use two decrementer sources. The idle decrementer + * constant is used when the last slow-poll detected no work to do, and the busy + * decrementer constant when the last slow-poll had work to do. */ +#define SLOW_POLL_IDLE 1000 +#define SLOW_POLL_BUSY 10 +static u32 __poll_portal_slow(struct bman_portal *p, u32 is); + +/* Portal interrupt handler */ +static irqreturn_t portal_isr(__always_unused int irq, void *ptr) +{ + struct bman_portal *p = ptr; + u32 clear = p->irq_sources; + u32 is = bm_isr_status_read(&p->p) & p->irq_sources; + + clear |= __poll_portal_slow(p, is); + bm_isr_status_clear(&p->p, clear); + return IRQ_HANDLED; +} + + +struct bman_portal *bman_create_portal( + struct bman_portal *portal, + const struct bm_portal_config *config) +{ + struct bm_portal *__p; + const struct bman_depletion *pools = &config->public_cfg.mask; + int ret; + u8 bpid = 0; + + if (!portal) { + portal = kmalloc(sizeof(*portal), GFP_KERNEL); + if (!portal) + return portal; + portal->alloced = 1; + } else + portal->alloced = 0; + + __p = &portal->p; + + /* prep the low-level portal struct with the mapped addresses from the + * config, everything that follows depends on it and "config" is more + * for (de)reference... */ + __p->addr.addr_ce = config->addr_virt[DPA_PORTAL_CE]; + __p->addr.addr_ci = config->addr_virt[DPA_PORTAL_CI]; + if (bm_rcr_init(__p, bm_rcr_pvb, bm_rcr_cce)) { + pr_err("RCR initialisation failed\n"); + goto fail_rcr; + } + if (bm_mc_init(__p)) { + pr_err("MC initialisation failed\n"); + goto fail_mc; + } + if (bm_isr_init(__p)) { + pr_err("ISR initialisation failed\n"); + goto fail_isr; + } + portal->pools = kmalloc(2 * sizeof(*pools), GFP_KERNEL); + if (!portal->pools) + goto fail_pools; + portal->pools[0] = *pools; + bman_depletion_init(portal->pools + 1); + while (bpid < bman_pool_max) { + /* Default to all BPIDs disabled, we enable as required at + * run-time. */ + bm_isr_bscn_mask(__p, bpid, 0); + bpid++; + } + portal->slowpoll = 0; +#ifdef FSL_DPA_CAN_WAIT_SYNC + portal->rcri_owned = NULL; +#endif +#ifdef FSL_DPA_PORTAL_SHARE + raw_spin_lock_init(&portal->sharing_lock); +#ifndef __rtems__ + portal->is_shared = config->public_cfg.is_shared; + portal->sharing_redirect = NULL; +#endif /* __rtems__ */ +#endif + memset(&portal->cb, 0, sizeof(portal->cb)); + /* Write-to-clear any stale interrupt status bits */ + bm_isr_disable_write(__p, 0xffffffff); + portal->irq_sources = 0; + bm_isr_enable_write(__p, portal->irq_sources); + bm_isr_status_clear(__p, 0xffffffff); + snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, config->public_cfg.cpu); + if (request_irq(config->public_cfg.irq, portal_isr, 0, portal->irqname, + portal)) { + pr_err("request_irq() failed\n"); + goto fail_irq; + } +#ifndef __rtems__ + if ((config->public_cfg.cpu != -1) && + irq_can_set_affinity(config->public_cfg.irq) && + irq_set_affinity(config->public_cfg.irq, + cpumask_of(config->public_cfg.cpu))) { + pr_err("irq_set_affinity() failed\n"); + goto fail_affinity; + } +#endif /* __rtems__ */ + + /* Need RCR to be empty before continuing */ + ret = bm_rcr_get_fill(__p); + if (ret) { + pr_err("RCR unclean\n"); + goto fail_rcr_empty; + } + /* Success */ + portal->config = config; + + bm_isr_disable_write(__p, 0); + bm_isr_uninhibit(__p); + return portal; +fail_rcr_empty: +#ifndef __rtems__ +fail_affinity: +#endif /* __rtems__ */ + free_irq(config->public_cfg.irq, portal); +fail_irq: + kfree(portal->pools); +fail_pools: + bm_isr_finish(__p); +fail_isr: + bm_mc_finish(__p); +fail_mc: + bm_rcr_finish(__p); +fail_rcr: + if (portal->alloced) + kfree(portal); + return NULL; +} + +struct bman_portal *bman_create_affine_portal( + const struct bm_portal_config *config) +{ + struct bman_portal *portal; + + portal = &per_cpu(bman_affine_portal, config->public_cfg.cpu); + portal = bman_create_portal(portal, config); +#ifndef __rtems__ + if (portal) { + spin_lock(&affine_mask_lock); + cpumask_set_cpu(config->public_cfg.cpu, &affine_mask); + spin_unlock(&affine_mask_lock); + } +#endif /* __rtems__ */ + return portal; +} + + +#ifndef __rtems__ +struct bman_portal *bman_create_affine_slave(struct bman_portal *redirect, + int cpu) +{ +#ifdef FSL_DPA_PORTAL_SHARE + struct bman_portal *p = &per_cpu(bman_affine_portal, cpu); + + BUG_ON(p->config); + BUG_ON(p->is_shared); + BUG_ON(!redirect->config->public_cfg.is_shared); + p->irq_sources = 0; + p->sharing_redirect = redirect; + put_affine_portal(); + return p; +#else + BUG(); + return NULL; +#endif +} +#endif /* __rtems__ */ + +void bman_destroy_portal(struct bman_portal *bm) +{ + const struct bm_portal_config *pcfg = bm->config; + + bm_rcr_cce_update(&bm->p); + bm_rcr_cce_update(&bm->p); + + free_irq(pcfg->public_cfg.irq, bm); + + kfree(bm->pools); + bm_isr_finish(&bm->p); + bm_mc_finish(&bm->p); + bm_rcr_finish(&bm->p); + bm->config = NULL; + if (bm->alloced) + kfree(bm); +} + +const struct bm_portal_config *bman_destroy_affine_portal(void) +{ + struct bman_portal *bm = get_raw_affine_portal(); + const struct bm_portal_config *pcfg; + +#ifdef FSL_DPA_PORTAL_SHARE +#ifndef __rtems__ + if (bm->sharing_redirect) { + bm->sharing_redirect = NULL; + put_affine_portal(); + return NULL; + } + bm->is_shared = 0; +#endif /* __rtems__ */ +#endif + pcfg = bm->config; + bman_destroy_portal(bm); +#ifndef __rtems__ + spin_lock(&affine_mask_lock); + cpumask_clear_cpu(pcfg->public_cfg.cpu, &affine_mask); + spin_unlock(&affine_mask_lock); +#endif /* __rtems__ */ + put_affine_portal(); + return pcfg; +} + +/* When release logic waits on available RCR space, we need a global waitqueue + * in the case of "affine" use (as the waits wake on different cpus which means + * different portals - so we can't wait on any per-portal waitqueue). */ +static DECLARE_WAIT_QUEUE_HEAD(affine_queue); + +static u32 __poll_portal_slow(struct bman_portal *p, u32 is) +{ + struct bman_depletion tmp; + u32 ret = is; + + /* There is a gotcha to be aware of. If we do the query before clearing + * the status register, we may miss state changes that occur between the + * two. If we write to clear the status register before the query, the + * cache-enabled query command may overtake the status register write + * unless we use a heavyweight sync (which we don't want). Instead, we + * write-to-clear the status register then *read it back* before doing + * the query, hence the odd while loop with the 'is' accumulation. */ + if (is & BM_PIRQ_BSCN) { + struct bm_mc_result *mcr; + __maybe_unused unsigned long irqflags; + unsigned int i, j; + u32 __is; + + bm_isr_status_clear(&p->p, BM_PIRQ_BSCN); + while ((__is = bm_isr_status_read(&p->p)) & BM_PIRQ_BSCN) { + is |= __is; + bm_isr_status_clear(&p->p, BM_PIRQ_BSCN); + } + is &= ~BM_PIRQ_BSCN; + PORTAL_IRQ_LOCK(p, irqflags); + bm_mc_start(&p->p); + bm_mc_commit(&p->p, BM_MCC_VERB_CMD_QUERY); + while (!(mcr = bm_mc_result(&p->p))) + cpu_relax(); + tmp = mcr->query.ds.state; + PORTAL_IRQ_UNLOCK(p, irqflags); + for (i = 0; i < 2; i++) { + int idx = i * 32; + /* tmp is a mask of currently-depleted pools. + * pools[0] is mask of those we care about. + * pools[1] is our previous view (we only want to + * be told about changes). */ + tmp.__state[i] &= p->pools[0].__state[i]; + if (tmp.__state[i] == p->pools[1].__state[i]) + /* fast-path, nothing to see, move along */ + continue; + for (j = 0; j <= 31; j++, idx++) { + struct bman_pool *pool = p->cb[idx]; + int b4 = bman_depletion_get(&p->pools[1], idx); + int af = bman_depletion_get(&tmp, idx); + + if (b4 == af) + continue; + while (pool) { + pool->params.cb(p, pool, + pool->params.cb_ctx, af); + pool = pool->next; + } + } + } + p->pools[1] = tmp; + } + + if (is & BM_PIRQ_RCRI) { + __maybe_unused unsigned long irqflags; + + PORTAL_IRQ_LOCK(p, irqflags); + bm_rcr_cce_update(&p->p); +#ifdef FSL_DPA_CAN_WAIT_SYNC + /* If waiting for sync, we only cancel the interrupt threshold + * when the ring utilisation hits zero. */ + if (p->rcri_owned) { + if (!bm_rcr_get_fill(&p->p)) { + p->rcri_owned = NULL; + bm_rcr_set_ithresh(&p->p, 0); + } + } else +#endif + bm_rcr_set_ithresh(&p->p, 0); + PORTAL_IRQ_UNLOCK(p, irqflags); + wake_up(&affine_queue); + bm_isr_status_clear(&p->p, BM_PIRQ_RCRI); + is &= ~BM_PIRQ_RCRI; + } + + /* There should be no status register bits left undefined */ + DPA_ASSERT(!is); + return ret; +} + +const struct bman_portal_config *bman_get_portal_config(void) +{ + struct bman_portal *p = get_affine_portal(); + const struct bman_portal_config *ret = &p->config->public_cfg; + + put_affine_portal(); + return ret; +} +EXPORT_SYMBOL(bman_get_portal_config); + +u32 bman_irqsource_get(void) +{ + struct bman_portal *p = get_raw_affine_portal(); + u32 ret = p->irq_sources & BM_PIRQ_VISIBLE; + + put_affine_portal(); + return ret; +} +EXPORT_SYMBOL(bman_irqsource_get); + +int bman_p_irqsource_add(struct bman_portal *p, __maybe_unused u32 bits) +{ + __maybe_unused unsigned long irqflags; + +#ifdef FSL_DPA_PORTAL_SHARE +#ifndef __rtems__ + if (p->sharing_redirect) + return -EINVAL; +#endif /* __rtems__ */ +#endif + PORTAL_IRQ_LOCK(p, irqflags); + set_bits(bits & BM_PIRQ_VISIBLE, &p->irq_sources); + bm_isr_enable_write(&p->p, p->irq_sources); + PORTAL_IRQ_UNLOCK(p, irqflags); + return 0; +} +EXPORT_SYMBOL(bman_p_irqsource_add); + +int bman_irqsource_add(__maybe_unused u32 bits) +{ + struct bman_portal *p = get_raw_affine_portal(); + int ret = bman_p_irqsource_add(p, bits); + + put_affine_portal(); + return ret; +} +EXPORT_SYMBOL(bman_irqsource_add); + +int bman_irqsource_remove(u32 bits) +{ + struct bman_portal *p = get_raw_affine_portal(); + __maybe_unused unsigned long irqflags; + u32 ier; + +#ifdef FSL_DPA_PORTAL_SHARE +#ifndef __rtems__ + if (p->sharing_redirect) { + put_affine_portal(); + return -EINVAL; + } +#endif /* __rtems__ */ +#endif + /* Our interrupt handler only processes+clears status register bits that + * are in p->irq_sources. As we're trimming that mask, if one of them + * were to assert in the status register just before we remove it from + * the enable register, there would be an interrupt-storm when we + * release the IRQ lock. So we wait for the enable register update to + * take effect in h/w (by reading it back) and then clear all other bits + * in the status register. Ie. we clear them from ISR once it's certain + * IER won't allow them to reassert. */ + PORTAL_IRQ_LOCK(p, irqflags); + bits &= BM_PIRQ_VISIBLE; + clear_bits(bits, &p->irq_sources); + bm_isr_enable_write(&p->p, p->irq_sources); + ier = bm_isr_enable_read(&p->p); + /* Using "~ier" (rather than "bits" or "~p->irq_sources") creates a + * data-dependency, ie. to protect against re-ordering. */ + bm_isr_status_clear(&p->p, ~ier); + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + return 0; +} +EXPORT_SYMBOL(bman_irqsource_remove); + +#ifndef __rtems__ +const cpumask_t *bman_affine_cpus(void) +{ + return &affine_mask; +} +EXPORT_SYMBOL(bman_affine_cpus); +#endif /* __rtems__ */ + +u32 bman_poll_slow(void) +{ + struct bman_portal *p = get_poll_portal(); + u32 ret; + +#ifdef FSL_DPA_PORTAL_SHARE +#ifndef __rtems__ + if (unlikely(p->sharing_redirect)) + ret = (u32)-1; + else +#endif /* __rtems__ */ +#endif + { + u32 is = bm_isr_status_read(&p->p) & ~p->irq_sources; + + ret = __poll_portal_slow(p, is); + bm_isr_status_clear(&p->p, ret); + } + put_poll_portal(); + return ret; +} +EXPORT_SYMBOL(bman_poll_slow); + +/* Legacy wrapper */ +void bman_poll(void) +{ + struct bman_portal *p = get_poll_portal(); + +#ifdef FSL_DPA_PORTAL_SHARE +#ifndef __rtems__ + if (unlikely(p->sharing_redirect)) + goto done; +#endif /* __rtems__ */ +#endif + if (!(p->slowpoll--)) { + u32 is = bm_isr_status_read(&p->p) & ~p->irq_sources; + u32 active = __poll_portal_slow(p, is); + + if (active) + p->slowpoll = SLOW_POLL_BUSY; + else + p->slowpoll = SLOW_POLL_IDLE; + } +#ifdef FSL_DPA_PORTAL_SHARE +#ifndef __rtems__ +done: +#endif /* __rtems__ */ +#endif + put_poll_portal(); +} +EXPORT_SYMBOL(bman_poll); + +static const u32 zero_thresholds[4] = {0, 0, 0, 0}; + +struct bman_pool *bman_new_pool(const struct bman_pool_params *params) +{ + struct bman_pool *pool = NULL; + u32 bpid; + + if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID) { + if (bman_alloc_bpid(&bpid)) + return NULL; + } else { + if (params->bpid >= bman_pool_max) + return NULL; + bpid = params->bpid; + } +#ifdef CONFIG_FSL_BMAN + if (params->flags & BMAN_POOL_FLAG_THRESH) { + if (bm_pool_set(bpid, params->thresholds)) + goto err; + } +#else + if (params->flags & BMAN_POOL_FLAG_THRESH) + goto err; +#endif + pool = kmalloc(sizeof(*pool), GFP_KERNEL); + if (!pool) + goto err; + pool->sp = NULL; + pool->sp_fill = 0; + pool->params = *params; +#ifdef CONFIG_FSL_DPA_CHECKING + atomic_set(&pool->in_use, 1); +#endif + if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID) + pool->params.bpid = bpid; + if (params->flags & BMAN_POOL_FLAG_STOCKPILE) { + pool->sp = kmalloc(sizeof(struct bm_buffer) * BMAN_STOCKPILE_SZ, + GFP_KERNEL); + if (!pool->sp) + goto err; + } + if (pool->params.flags & BMAN_POOL_FLAG_DEPLETION) { + struct bman_portal *p = get_affine_portal(); + + if (!p->pools || !bman_depletion_get(&p->pools[0], bpid)) { + pr_err("Depletion events disabled for bpid %d\n", bpid); + goto err; + } + depletion_link(p, pool); + put_affine_portal(); + } + return pool; +err: +#ifdef CONFIG_FSL_BMAN + if (params->flags & BMAN_POOL_FLAG_THRESH) + bm_pool_set(bpid, zero_thresholds); +#endif + if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID) + bman_release_bpid(bpid); + if (pool) { + kfree(pool->sp); + kfree(pool); + } + return NULL; +} +EXPORT_SYMBOL(bman_new_pool); + +void bman_free_pool(struct bman_pool *pool) +{ +#ifdef CONFIG_FSL_BMAN + if (pool->params.flags & BMAN_POOL_FLAG_THRESH) + bm_pool_set(pool->params.bpid, zero_thresholds); +#endif + if (pool->params.flags & BMAN_POOL_FLAG_DEPLETION) + depletion_unlink(pool); + if (pool->params.flags & BMAN_POOL_FLAG_STOCKPILE) { + if (pool->sp_fill) + pr_err("Stockpile not flushed, has %u in bpid %u.\n", + pool->sp_fill, pool->params.bpid); + kfree(pool->sp); + pool->sp = NULL; + pool->params.flags ^= BMAN_POOL_FLAG_STOCKPILE; + } + if (pool->params.flags & BMAN_POOL_FLAG_DYNAMIC_BPID) + bman_release_bpid(pool->params.bpid); + kfree(pool); +} +EXPORT_SYMBOL(bman_free_pool); + +const struct bman_pool_params *bman_get_params(const struct bman_pool *pool) +{ + return &pool->params; +} +EXPORT_SYMBOL(bman_get_params); + +static noinline void update_rcr_ci(struct bman_portal *p, u8 avail) +{ + if (avail) + bm_rcr_cce_prefetch(&p->p); + else + bm_rcr_cce_update(&p->p); +} + +int bman_rcr_is_empty(void) +{ + __maybe_unused unsigned long irqflags; + struct bman_portal *p = get_affine_portal(); + u8 avail; + + PORTAL_IRQ_LOCK(p, irqflags); + update_rcr_ci(p, 0); + avail = bm_rcr_get_fill(&p->p); + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + return avail == 0; +} +EXPORT_SYMBOL(bman_rcr_is_empty); + +static inline struct bm_rcr_entry *try_rel_start(struct bman_portal **p, +#ifdef FSL_DPA_CAN_WAIT + __maybe_unused struct bman_pool *pool, +#endif + __maybe_unused unsigned long *irqflags, + __maybe_unused u32 flags) +{ + struct bm_rcr_entry *r; + u8 avail; + + *p = get_affine_portal(); + PORTAL_IRQ_LOCK(*p, (*irqflags)); +#ifdef FSL_DPA_CAN_WAIT_SYNC + if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) && + (flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) { + if ((*p)->rcri_owned) { + PORTAL_IRQ_UNLOCK(*p, (*irqflags)); + put_affine_portal(); + return NULL; + } + (*p)->rcri_owned = pool; + } +#endif + avail = bm_rcr_get_avail(&(*p)->p); + if (avail < 2) + update_rcr_ci(*p, avail); + r = bm_rcr_start(&(*p)->p); + if (unlikely(!r)) { +#ifdef FSL_DPA_CAN_WAIT_SYNC + if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) && + (flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) + (*p)->rcri_owned = NULL; +#endif + PORTAL_IRQ_UNLOCK(*p, (*irqflags)); + put_affine_portal(); + } + return r; +} + +#ifdef FSL_DPA_CAN_WAIT +static noinline struct bm_rcr_entry *__wait_rel_start(struct bman_portal **p, + struct bman_pool *pool, + __maybe_unused unsigned long *irqflags, + u32 flags) +{ + struct bm_rcr_entry *rcr = try_rel_start(p, pool, irqflags, flags); + + if (!rcr) + bm_rcr_set_ithresh(&(*p)->p, 1); + return rcr; +} + +static noinline struct bm_rcr_entry *wait_rel_start(struct bman_portal **p, + struct bman_pool *pool, + __maybe_unused unsigned long *irqflags, + u32 flags) +{ + struct bm_rcr_entry *rcr; +#ifndef FSL_DPA_CAN_WAIT_SYNC + pool = NULL; +#endif +#ifndef __rtems__ + if (flags & BMAN_RELEASE_FLAG_WAIT_INT) + wait_event_interruptible(affine_queue, + (rcr = __wait_rel_start(p, pool, irqflags, flags))); + else +#endif /* __rtems__ */ + wait_event(affine_queue, + (rcr = __wait_rel_start(p, pool, irqflags, flags))); + return rcr; +} +#endif + +/* to facilitate better copying of bufs into the ring without either (a) copying + * noise into the first byte (prematurely triggering the command), nor (b) being + * very inefficient by copying small fields using read-modify-write */ +struct overlay_bm_buffer { + u32 first; + u32 second; +}; + +static inline int __bman_release(struct bman_pool *pool, + const struct bm_buffer *bufs, u8 num, u32 flags) +{ + struct bman_portal *p; + struct bm_rcr_entry *r; + struct overlay_bm_buffer *o_dest; + struct overlay_bm_buffer *o_src = (struct overlay_bm_buffer *)&bufs[0]; + __maybe_unused unsigned long irqflags; + u32 i = num - 1; + +#ifdef FSL_DPA_CAN_WAIT + if (flags & BMAN_RELEASE_FLAG_WAIT) + r = wait_rel_start(&p, pool, &irqflags, flags); + else + r = try_rel_start(&p, pool, &irqflags, flags); +#else + r = try_rel_start(&p, &irqflags, flags); +#endif + if (!r) + return -EBUSY; + /* We can copy all but the first entry, as this can trigger badness + * with the valid-bit. Use the overlay to mask the verb byte. */ + o_dest = (struct overlay_bm_buffer *)&r->bufs[0]; + o_dest->first = (o_src->first & 0x0000ffff) | + (((u32)pool->params.bpid << 16) & 0x00ff0000); + o_dest->second = o_src->second; + if (i) + copy_words(&r->bufs[1], &bufs[1], i * sizeof(bufs[0])); + bm_rcr_pvb_commit(&p->p, BM_RCR_VERB_CMD_BPID_SINGLE | + (num & BM_RCR_VERB_BUFCOUNT_MASK)); +#ifdef FSL_DPA_CAN_WAIT_SYNC + /* if we wish to sync we need to set the threshold after h/w sees the + * new ring entry. As we're mixing cache-enabled and cache-inhibited + * accesses, this requires a heavy-weight sync. */ + if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) && + (flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) { + hwsync(); + bm_rcr_set_ithresh(&p->p, 1); + } +#endif + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); +#ifdef FSL_DPA_CAN_WAIT_SYNC + if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) && + (flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) { +#ifndef __rtems__ + if (flags & BMAN_RELEASE_FLAG_WAIT_INT) + wait_event_interruptible(affine_queue, + (p->rcri_owned != pool)); + else +#endif /* __rtems__ */ + wait_event(affine_queue, (p->rcri_owned != pool)); + } +#endif + return 0; +} + +int bman_release(struct bman_pool *pool, const struct bm_buffer *bufs, u8 num, + u32 flags) +{ + int ret = 0; + +#ifdef CONFIG_FSL_DPA_CHECKING + if (!num || (num > 8)) + return -EINVAL; + if (pool->params.flags & BMAN_POOL_FLAG_NO_RELEASE) + return -EINVAL; +#endif + /* Without stockpile, this API is a pass-through to the h/w operation */ + if (!(pool->params.flags & BMAN_POOL_FLAG_STOCKPILE)) + return __bman_release(pool, bufs, num, flags); +#ifdef CONFIG_FSL_DPA_CHECKING + if (!atomic_dec_and_test(&pool->in_use)) { + pr_crit("Parallel attempts to enter bman_released() detected."); + panic("only one instance of bman_released/acquired allowed"); + } +#endif + /* This needs some explanation. Adding the given buffers may take the + * stockpile over the threshold, but in fact the stockpile may already + * *be* over the threshold if a previous release-to-hw attempt had + * failed. So we have 3 cases to cover; + * 1. we add to the stockpile and don't hit the threshold, + * 2. we add to the stockpile, hit the threshold and release-to-hw, + * 3. we have to release-to-hw before adding to the stockpile + * (not enough room in the stockpile for case 2). + * Our constraints on thresholds guarantee that in case 3, there must be + * at least 8 bufs already in the stockpile, so all release-to-hw ops + * are for 8 bufs. Despite all this, the API must indicate whether the + * given buffers were taken off the caller's hands, irrespective of + * whether a release-to-hw was attempted. */ + while (num) { + /* Add buffers to stockpile if they fit */ + if ((pool->sp_fill + num) < BMAN_STOCKPILE_SZ) { + copy_words(pool->sp + pool->sp_fill, bufs, + sizeof(struct bm_buffer) * num); + pool->sp_fill += num; + num = 0; /* --> will return success no matter what */ + } + /* Do hw op if hitting the high-water threshold */ + if ((pool->sp_fill + num) >= BMAN_STOCKPILE_HIGH) { + ret = __bman_release(pool, + pool->sp + (pool->sp_fill - 8), 8, flags); + if (ret) { + ret = (num ? ret : 0); + goto release_done; + } + pool->sp_fill -= 8; + } + } +release_done: +#ifdef CONFIG_FSL_DPA_CHECKING + atomic_inc(&pool->in_use); +#endif + return ret; +} +EXPORT_SYMBOL(bman_release); + +static inline int __bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, + u8 num) +{ + struct bman_portal *p = get_affine_portal(); + struct bm_mc_command *mcc; + struct bm_mc_result *mcr; + __maybe_unused unsigned long irqflags; + int ret; + + PORTAL_IRQ_LOCK(p, irqflags); + mcc = bm_mc_start(&p->p); + mcc->acquire.bpid = pool->params.bpid; + bm_mc_commit(&p->p, BM_MCC_VERB_CMD_ACQUIRE | + (num & BM_MCC_VERB_ACQUIRE_BUFCOUNT)); + while (!(mcr = bm_mc_result(&p->p))) + cpu_relax(); + ret = mcr->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT; + if (bufs) + copy_words(&bufs[0], &mcr->acquire.bufs[0], + num * sizeof(bufs[0])); + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + if (ret != num) + ret = -ENOMEM; + return ret; +} + +int bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, u8 num, + u32 flags) +{ + int ret = 0; + +#ifdef CONFIG_FSL_DPA_CHECKING + if (!num || (num > 8)) + return -EINVAL; + if (pool->params.flags & BMAN_POOL_FLAG_ONLY_RELEASE) + return -EINVAL; +#endif + /* Without stockpile, this API is a pass-through to the h/w operation */ + if (!(pool->params.flags & BMAN_POOL_FLAG_STOCKPILE)) + return __bman_acquire(pool, bufs, num); +#ifdef CONFIG_FSL_DPA_CHECKING + if (!atomic_dec_and_test(&pool->in_use)) { + pr_crit("Parallel attempts to enter bman_acquire() detected."); + panic("only one instance of bman_released/acquired allowed"); + } +#endif + /* Only need a h/w op if we'll hit the low-water thresh */ + if (!(flags & BMAN_ACQUIRE_FLAG_STOCKPILE) && + (pool->sp_fill <= (BMAN_STOCKPILE_LOW + num))) { + /* refill stockpile with max amount, but if max amount + * isn't available, try amount the user wants */ + int bufcount = 8; + + ret = __bman_acquire(pool, pool->sp + pool->sp_fill, bufcount); + if (ret < 0 && bufcount != num) { + bufcount = num; + /* Maybe buffer pool has less than 8 */ + ret = __bman_acquire(pool, pool->sp + pool->sp_fill, + bufcount); + } + if (ret < 0) + goto hw_starved; + DPA_ASSERT(ret == bufcount); + pool->sp_fill += bufcount; + } else { +hw_starved: + if (pool->sp_fill < num) { + ret = -ENOMEM; + goto acquire_done; + } + } + copy_words(bufs, pool->sp + (pool->sp_fill - num), + sizeof(struct bm_buffer) * num); + pool->sp_fill -= num; + ret = num; +acquire_done: +#ifdef CONFIG_FSL_DPA_CHECKING + atomic_inc(&pool->in_use); +#endif + return ret; +} +EXPORT_SYMBOL(bman_acquire); + +int bman_flush_stockpile(struct bman_pool *pool, u32 flags) +{ + u8 num; + int ret; + + while (pool->sp_fill) { + num = ((pool->sp_fill > 8) ? 8 : pool->sp_fill); + ret = __bman_release(pool, pool->sp + (pool->sp_fill - num), + num, flags); + if (ret) + return ret; + pool->sp_fill -= num; + } + return 0; +} +EXPORT_SYMBOL(bman_flush_stockpile); + +int bman_query_pools(struct bm_pool_state *state) +{ + struct bman_portal *p = get_affine_portal(); + struct bm_mc_result *mcr; + __maybe_unused unsigned long irqflags; + + PORTAL_IRQ_LOCK(p, irqflags); + bm_mc_start(&p->p); + bm_mc_commit(&p->p, BM_MCC_VERB_CMD_QUERY); + while (!(mcr = bm_mc_result(&p->p))) + cpu_relax(); + DPA_ASSERT((mcr->verb & BM_MCR_VERB_CMD_MASK) == BM_MCR_VERB_CMD_QUERY); + *state = mcr->query; + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + return 0; +} +EXPORT_SYMBOL(bman_query_pools); + +#ifdef CONFIG_FSL_BMAN +u32 bman_query_free_buffers(struct bman_pool *pool) +{ + return bm_pool_free_buffers(pool->params.bpid); +} +EXPORT_SYMBOL(bman_query_free_buffers); + +int bman_update_pool_thresholds(struct bman_pool *pool, const u32 *thresholds) +{ + u32 bpid; + + bpid = bman_get_params(pool)->bpid; + + return bm_pool_set(bpid, thresholds); +} +EXPORT_SYMBOL(bman_update_pool_thresholds); +#endif + +int bman_shutdown_pool(u32 bpid) +{ + struct bman_portal *p = get_affine_portal(); + __maybe_unused unsigned long irqflags; + int ret; + + PORTAL_IRQ_LOCK(p, irqflags); + ret = bm_shutdown_pool(&p->p, bpid); + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + return ret; +} +EXPORT_SYMBOL(bman_shutdown_pool); + +const struct bm_portal_config * +bman_get_bm_portal_config(const struct bman_portal *portal) +{ +#ifndef __rtems__ + return portal->sharing_redirect ? NULL : portal->config; +#else /* __rtems__ */ + return portal->config; +#endif /* __rtems__ */ +} |