path: root/linux/drivers/soc/fsl/qbman/bman_api.c

#include <machine/rtems-bsd-kernel-space.h>

#include <rtems/bsd/local/opt_dpaa.h>

/* Copyright 2008 - 2015 Freescale Semiconductor, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *	 notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *	 notice, this list of conditions and the following disclaimer in the
 *	 documentation and/or other materials provided with the distribution.
 *     * Neither the name of Freescale Semiconductor nor the
 *	 names of its contributors may be used to endorse or promote products
 *	 derived from this software without specific prior written permission.
 *
 * ALTERNATIVELY, this software may be distributed under the terms of the
 * GNU General Public License ("GPL") as published by the Free Software
 * Foundation, either version 2 of that License or (at your option) any
 * later version.
 *
 * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "bman.h"

/* Compilation constants */
#define RCR_THRESH	2	/* reread h/w CI when running out of space */
#define IRQNAME		"BMan portal %d"
#define MAX_IRQNAME	16	/* big enough for "BMan portal %d" */
#define FSL_DPA_PORTAL_SHARE 1  /* Allow portals to be shared */

struct bman_portal {
	struct bm_portal p;
	/* 2-element array. pools[0] is mask, pools[1] is snapshot. */
	struct bman_depletion *pools;
	int thresh_set;
	unsigned long irq_sources;
	u32 slowpoll;	/* only used when interrupts are off */
#ifdef FSL_DPA_CAN_WAIT_SYNC
	struct bman_pool *rcri_owned; /* only 1 release WAIT_SYNC at a time */
#endif
#ifdef FSL_DPA_PORTAL_SHARE
	raw_spinlock_t sharing_lock; /* only used if is_shared */
#ifndef __rtems__
	int is_shared;
	struct bman_portal *sharing_redirect;
#endif /* __rtems__ */
#endif
	/* When the cpu-affine portal is activated, this is non-NULL */
	const struct bm_portal_config *config;
	/* 64-entry hash-table of pool objects that are tracking depletion
	 * entry/exit (ie. BMAN_POOL_FLAG_DEPLETION). This isn't fast-path, so
	 * we're not fussy about cache-misses and so forth - whereas the above
	 * members should all fit in one cacheline.
	 * BTW, with 64 entries in the hash table and 64 buffer pools to track,
	 * you'll never guess the hash-function ... */
	struct bman_pool *cb[64];
	char irqname[MAX_IRQNAME];
	/* Track if the portal was alloced by the driver */
	u8 alloced;
};


#ifdef FSL_DPA_PORTAL_SHARE
/* For an explanation of the locking, redirection, or affine-portal logic,
 * please consult the QMan driver for details. This is the same, only simpler
 * (no fiddly QMan-specific bits.) */
#ifndef __rtems__
#define PORTAL_IRQ_LOCK(p, irqflags) \
	do { \
		if ((p)->is_shared) \
			raw_spin_lock_irqsave(&(p)->sharing_lock, irqflags); \
		else \
			local_irq_save(irqflags); \
	} while (0)
#define PORTAL_IRQ_UNLOCK(p, irqflags) \
	do { \
		if ((p)->is_shared) \
			raw_spin_unlock_irqrestore(&(p)->sharing_lock, \
						   irqflags); \
		else \
			local_irq_restore(irqflags); \
	} while (0)
#else /* __rtems__ */
#define PORTAL_IRQ_LOCK(p, irqflags) \
    raw_spin_lock_irqsave(&(p)->sharing_lock, irqflags)
#define PORTAL_IRQ_UNLOCK(p, irqflags) \
    raw_spin_unlock_irqrestore(&(p)->sharing_lock, irqflags)
#endif /* __rtems__ */
#else
#define PORTAL_IRQ_LOCK(p, irqflags) local_irq_save(irqflags)
#define PORTAL_IRQ_UNLOCK(p, irqflags) local_irq_restore(irqflags)
#endif

#ifndef __rtems__
static cpumask_t affine_mask;
static DEFINE_SPINLOCK(affine_mask_lock);
#endif /* __rtems__ */
static DEFINE_PER_CPU(struct bman_portal, bman_affine_portal);
static inline struct bman_portal *get_raw_affine_portal(void)
{
	return &get_cpu_var(bman_affine_portal);
}
#ifdef FSL_DPA_PORTAL_SHARE
static inline struct bman_portal *get_affine_portal(void)
{
	struct bman_portal *p = get_raw_affine_portal();

#ifndef __rtems__
	if (p->sharing_redirect)
		return p->sharing_redirect;
#endif /* __rtems__ */
	return p;
}
#else
#define get_affine_portal() get_raw_affine_portal()
#endif
static inline void put_affine_portal(void)
{
	put_cpu_var(bman_affine_portal);
}
static inline struct bman_portal *get_poll_portal(void)
{
	return this_cpu_ptr(&bman_affine_portal);
}
#define put_poll_portal()

/* GOTCHA: this object type refers to a pool, it isn't *the* pool. There may be
 * more than one such object per BMan buffer pool, eg. if different users of the
 * pool are operating via different portals. */
struct bman_pool {
	struct bman_pool_params params;
	/* Used for hash-table admin when using depletion notifications. */
	struct bman_portal *portal;
	struct bman_pool *next;
	/* stockpile state - NULL unless BMAN_POOL_FLAG_STOCKPILE is set */
	struct bm_buffer *sp;
	unsigned int sp_fill;
#ifdef CONFIG_FSL_DPA_CHECKING
	atomic_t in_use;
#endif
};

/* (De)Registration of depletion notification callbacks */
static void depletion_link(struct bman_portal *portal, struct bman_pool *pool)
{
	__maybe_unused unsigned long irqflags;

	pool->portal = portal;
	PORTAL_IRQ_LOCK(portal, irqflags);
	pool->next = portal->cb[pool->params.bpid];
	portal->cb[pool->params.bpid] = pool;
	if (!pool->next)
		/* First object for that bpid on this portal, enable the BSCN
		 * mask bit. */
		bm_isr_bscn_mask(&portal->p, pool->params.bpid, 1);
	PORTAL_IRQ_UNLOCK(portal, irqflags);
}
static void depletion_unlink(struct bman_pool *pool)
{
	struct bman_pool *it, *last = NULL;
	struct bman_pool **base = &pool->portal->cb[pool->params.bpid];
	__maybe_unused unsigned long irqflags;

	PORTAL_IRQ_LOCK(pool->portal, irqflags);
	it = *base;	/* <-- gotcha, don't do this prior to the irq_save */
	while (it != pool) {
		last = it;
		it = it->next;
	}
	if (!last)
		*base = pool->next;
	else
		last->next = pool->next;
	if (!last && !pool->next) {
		/* Last object for that bpid on this portal, disable the BSCN
		 * mask bit. */
		bm_isr_bscn_mask(&pool->portal->p, pool->params.bpid, 0);
		/* And "forget" that we last saw this pool as depleted */
		bman_depletion_unset(&pool->portal->pools[1],
					pool->params.bpid);
	}
	PORTAL_IRQ_UNLOCK(pool->portal, irqflags);
}

/* In the case that the application's core loop calls qman_poll() and
 * bman_poll(), we ought to balance how often we incur the overheads of the
 * slow-path poll. We'll use two decrementer sources. The idle decrementer
 * constant is used when the last slow-poll detected no work to do, and the busy
 * decrementer constant when the last slow-poll had work to do. */
#define SLOW_POLL_IDLE 1000
#define SLOW_POLL_BUSY 10
static u32 __poll_portal_slow(struct bman_portal *p, u32 is);

/* Portal interrupt handler */
static irqreturn_t portal_isr(__always_unused int irq, void *ptr)
{
	struct bman_portal *p = ptr;
	u32 clear = p->irq_sources;
	u32 is = bm_isr_status_read(&p->p) & p->irq_sources;

	clear |= __poll_portal_slow(p, is);
	bm_isr_status_clear(&p->p, clear);
	return IRQ_HANDLED;
}


struct bman_portal *bman_create_portal(
				       struct bman_portal *portal,
				       const struct bm_portal_config *config)
{
	struct bm_portal *__p;
	const struct bman_depletion *pools = &config->public_cfg.mask;
	int ret;
	u8 bpid = 0;

	if (!portal) {
		portal = kmalloc(sizeof(*portal), GFP_KERNEL);
		if (!portal)
			return portal;
		portal->alloced = 1;
	} else
		portal->alloced = 0;

	__p = &portal->p;

	/* prep the low-level portal struct with the mapped addresses from the
	 * config, everything that follows depends on it and "config" is more
	 * for (de)reference... */
	__p->addr.addr_ce = config->addr_virt[DPA_PORTAL_CE];
	__p->addr.addr_ci = config->addr_virt[DPA_PORTAL_CI];
	if (bm_rcr_init(__p, bm_rcr_pvb, bm_rcr_cce)) {
		pr_err("RCR initialisation failed\n");
		goto fail_rcr;
	}
	if (bm_mc_init(__p)) {
		pr_err("MC initialisation failed\n");
		goto fail_mc;
	}
	if (bm_isr_init(__p)) {
		pr_err("ISR initialisation failed\n");
		goto fail_isr;
	}
	portal->pools = kmalloc(2 * sizeof(*pools), GFP_KERNEL);
	if (!portal->pools)
		goto fail_pools;
	portal->pools[0] = *pools;
	bman_depletion_init(portal->pools + 1);
	while (bpid < bman_pool_max) {
		/* Default to all BPIDs disabled, we enable as required at
		 * run-time. */
		bm_isr_bscn_mask(__p, bpid, 0);
		bpid++;
	}
	portal->slowpoll = 0;
#ifdef FSL_DPA_CAN_WAIT_SYNC
	portal->rcri_owned = NULL;
#endif
#ifdef FSL_DPA_PORTAL_SHARE
	raw_spin_lock_init(&portal->sharing_lock);
#ifndef __rtems__
	portal->is_shared = config->public_cfg.is_shared;
	portal->sharing_redirect = NULL;
#endif /* __rtems__ */
#endif
	memset(&portal->cb, 0, sizeof(portal->cb));
	/* Write-to-clear any stale interrupt status bits */
	bm_isr_disable_write(__p, 0xffffffff);
	portal->irq_sources = 0;
	bm_isr_enable_write(__p, portal->irq_sources);
	bm_isr_status_clear(__p, 0xffffffff);
	snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, config->public_cfg.cpu);
	if (request_irq(config->public_cfg.irq, portal_isr, 0, portal->irqname,
			portal)) {
		pr_err("request_irq() failed\n");
		goto fail_irq;
	}
#ifndef __rtems__
	if ((config->public_cfg.cpu != -1) &&
			irq_can_set_affinity(config->public_cfg.irq) &&
			irq_set_affinity(config->public_cfg.irq,
				cpumask_of(config->public_cfg.cpu))) {
		pr_err("irq_set_affinity() failed\n");
		goto fail_affinity;
	}
#endif /* __rtems__ */

	/* Need RCR to be empty before continuing */
	ret = bm_rcr_get_fill(__p);
	if (ret) {
		pr_err("RCR unclean\n");
		goto fail_rcr_empty;
	}
	/* Success */
	portal->config = config;

	bm_isr_disable_write(__p, 0);
	bm_isr_uninhibit(__p);
	return portal;
fail_rcr_empty:
#ifndef __rtems__
fail_affinity:
#endif /* __rtems__ */
	free_irq(config->public_cfg.irq, portal);
fail_irq:
	kfree(portal->pools);
fail_pools:
	bm_isr_finish(__p);
fail_isr:
	bm_mc_finish(__p);
fail_mc:
	bm_rcr_finish(__p);
fail_rcr:
	if (portal->alloced)
		kfree(portal);
	return NULL;
}

struct bman_portal *bman_create_affine_portal(
			const struct bm_portal_config *config)
{
	struct bman_portal *portal;

	portal = &per_cpu(bman_affine_portal, config->public_cfg.cpu);
	portal = bman_create_portal(portal, config);
#ifndef __rtems__
	if (portal) {
		spin_lock(&affine_mask_lock);
		cpumask_set_cpu(config->public_cfg.cpu, &affine_mask);
		spin_unlock(&affine_mask_lock);
	}
#endif /* __rtems__ */
	return portal;
}


#ifndef __rtems__
struct bman_portal *bman_create_affine_slave(struct bman_portal *redirect,
								int cpu)
{
#ifdef FSL_DPA_PORTAL_SHARE
	struct bman_portal *p = &per_cpu(bman_affine_portal, cpu);

	BUG_ON(p->config);
	BUG_ON(p->is_shared);
	BUG_ON(!redirect->config->public_cfg.is_shared);
	p->irq_sources = 0;
	p->sharing_redirect = redirect;
	put_affine_portal();
	return p;
#else
	BUG();
	return NULL;
#endif
}
#endif /* __rtems__ */

void bman_destroy_portal(struct bman_portal *bm)
{
	const struct bm_portal_config *pcfg = bm->config;

	bm_rcr_cce_update(&bm->p);
	bm_rcr_cce_update(&bm->p);

	free_irq(pcfg->public_cfg.irq, bm);

	kfree(bm->pools);
	bm_isr_finish(&bm->p);
	bm_mc_finish(&bm->p);
	bm_rcr_finish(&bm->p);
	bm->config = NULL;
	if (bm->alloced)
		kfree(bm);
}

const struct bm_portal_config *bman_destroy_affine_portal(void)
{
	struct bman_portal *bm = get_raw_affine_portal();
	const struct bm_portal_config *pcfg;

#ifdef FSL_DPA_PORTAL_SHARE
#ifndef __rtems__
	if (bm->sharing_redirect) {
		bm->sharing_redirect = NULL;
		put_affine_portal();
		return NULL;
	}
	bm->is_shared = 0;
#endif /* __rtems__ */
#endif
	pcfg = bm->config;
	bman_destroy_portal(bm);
#ifndef __rtems__
	spin_lock(&affine_mask_lock);
	cpumask_clear_cpu(pcfg->public_cfg.cpu, &affine_mask);
	spin_unlock(&affine_mask_lock);
#endif /* __rtems__ */
	put_affine_portal();
	return pcfg;
}

/* When release logic waits on available RCR space, we need a global waitqueue
 * in the case of "affine" use (as the waits wake on different cpus which means
 * different portals - so we can't wait on any per-portal waitqueue). */
static DECLARE_WAIT_QUEUE_HEAD(affine_queue);

static u32 __poll_portal_slow(struct bman_portal *p, u32 is)
{
	struct bman_depletion tmp;
	u32 ret = is;

	/* There is a gotcha to be aware of. If we do the query before clearing
	 * the status register, we may miss state changes that occur between the
	 * two. If we write to clear the status register before the query, the
	 * cache-enabled query command may overtake the status register write
	 * unless we use a heavyweight sync (which we don't want). Instead, we
	 * write-to-clear the status register then *read it back* before doing
	 * the query, hence the odd while loop with the 'is' accumulation. */
	if (is & BM_PIRQ_BSCN) {
		struct bm_mc_result *mcr;
		__maybe_unused unsigned long irqflags;
		unsigned int i, j;
		u32 __is;

		bm_isr_status_clear(&p->p, BM_PIRQ_BSCN);
		while ((__is = bm_isr_status_read(&p->p)) & BM_PIRQ_BSCN) {
			is |= __is;
			bm_isr_status_clear(&p->p, BM_PIRQ_BSCN);
		}
		is &= ~BM_PIRQ_BSCN;
		PORTAL_IRQ_LOCK(p, irqflags);
		bm_mc_start(&p->p);
		bm_mc_commit(&p->p, BM_MCC_VERB_CMD_QUERY);
		while (!(mcr = bm_mc_result(&p->p)))
			cpu_relax();
		tmp = mcr->query.ds.state;
		PORTAL_IRQ_UNLOCK(p, irqflags);
		for (i = 0; i < 2; i++) {
			int idx = i * 32;
			/* tmp is a mask of currently-depleted pools.
			 * pools[0] is mask of those we care about.
			 * pools[1] is our previous view (we only want to
			 * be told about changes). */
			tmp.__state[i] &= p->pools[0].__state[i];
			if (tmp.__state[i] == p->pools[1].__state[i])
				/* fast-path, nothing to see, move along */
				continue;
			for (j = 0; j <= 31; j++, idx++) {
				struct bman_pool *pool = p->cb[idx];
				int b4 = bman_depletion_get(&p->pools[1], idx);
				int af = bman_depletion_get(&tmp, idx);

				if (b4 == af)
					continue;
				while (pool) {
					pool->params.cb(p, pool,
						pool->params.cb_ctx, af);
					pool = pool->next;
				}
			}
		}
		p->pools[1] = tmp;
	}

	if (is & BM_PIRQ_RCRI) {
		__maybe_unused unsigned long irqflags;

		PORTAL_IRQ_LOCK(p, irqflags);
		bm_rcr_cce_update(&p->p);
#ifdef FSL_DPA_CAN_WAIT_SYNC
		/* If waiting for sync, we only cancel the interrupt threshold
		 * when the ring utilisation hits zero. */
		if (p->rcri_owned) {
			if (!bm_rcr_get_fill(&p->p)) {
				p->rcri_owned = NULL;
				bm_rcr_set_ithresh(&p->p, 0);
			}
		} else
#endif
		bm_rcr_set_ithresh(&p->p, 0);
		PORTAL_IRQ_UNLOCK(p, irqflags);
		wake_up(&affine_queue);
		bm_isr_status_clear(&p->p, BM_PIRQ_RCRI);
		is &= ~BM_PIRQ_RCRI;
	}

	/* There should be no status register bits left undefined */
	DPA_ASSERT(!is);
	return ret;
}

const struct bman_portal_config *bman_get_portal_config(void)
{
	struct bman_portal *p = get_affine_portal();
	const struct bman_portal_config *ret = &p->config->public_cfg;

	put_affine_portal();
	return ret;
}
EXPORT_SYMBOL(bman_get_portal_config);

u32 bman_irqsource_get(void)
{
	struct bman_portal *p = get_raw_affine_portal();
	u32 ret = p->irq_sources & BM_PIRQ_VISIBLE;

	put_affine_portal();
	return ret;
}
EXPORT_SYMBOL(bman_irqsource_get);

int bman_p_irqsource_add(struct bman_portal *p, __maybe_unused u32 bits)
{
	__maybe_unused unsigned long irqflags;

#ifdef FSL_DPA_PORTAL_SHARE
#ifndef __rtems__
	if (p->sharing_redirect)
		return -EINVAL;
#endif /* __rtems__ */
#endif
	PORTAL_IRQ_LOCK(p, irqflags);
	set_bits(bits & BM_PIRQ_VISIBLE, &p->irq_sources);
	bm_isr_enable_write(&p->p, p->irq_sources);
	PORTAL_IRQ_UNLOCK(p, irqflags);
	return 0;
}
EXPORT_SYMBOL(bman_p_irqsource_add);

int bman_irqsource_add(__maybe_unused u32 bits)
{
	struct bman_portal *p = get_raw_affine_portal();
	int ret = bman_p_irqsource_add(p, bits);

	put_affine_portal();
	return ret;
}
EXPORT_SYMBOL(bman_irqsource_add);

int bman_irqsource_remove(u32 bits)
{
	struct bman_portal *p = get_raw_affine_portal();
	__maybe_unused unsigned long irqflags;
	u32 ier;

#ifdef FSL_DPA_PORTAL_SHARE
#ifndef __rtems__
	if (p->sharing_redirect) {
		put_affine_portal();
		return -EINVAL;
	}
#endif /* __rtems__ */
#endif
	/* Our interrupt handler only processes+clears status register bits that
	 * are in p->irq_sources. As we're trimming that mask, if one of them
	 * were to assert in the status register just before we remove it from
	 * the enable register, there would be an interrupt-storm when we
	 * release the IRQ lock. So we wait for the enable register update to
	 * take effect in h/w (by reading it back) and then clear all other bits
	 * in the status register. Ie. we clear them from ISR once it's certain
	 * IER won't allow them to reassert. */
	PORTAL_IRQ_LOCK(p, irqflags);
	bits &= BM_PIRQ_VISIBLE;
	clear_bits(bits, &p->irq_sources);
	bm_isr_enable_write(&p->p, p->irq_sources);
	ier = bm_isr_enable_read(&p->p);
	/* Using "~ier" (rather than "bits" or "~p->irq_sources") creates a
	 * data-dependency, ie. to protect against re-ordering. */
	bm_isr_status_clear(&p->p, ~ier);
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
	return 0;
}
EXPORT_SYMBOL(bman_irqsource_remove);

#ifndef __rtems__
const cpumask_t *bman_affine_cpus(void)
{
	return &affine_mask;
}
EXPORT_SYMBOL(bman_affine_cpus);
#endif /* __rtems__ */

u32 bman_poll_slow(void)
{
	struct bman_portal *p = get_poll_portal();
	u32 ret;

#ifdef FSL_DPA_PORTAL_SHARE
#ifndef __rtems__
	if (unlikely(p->sharing_redirect))
		ret = (u32)-1;
	else
#endif /* __rtems__ */
#endif
	{
		u32 is = bm_isr_status_read(&p->p) & ~p->irq_sources;

		ret = __poll_portal_slow(p, is);
		bm_isr_status_clear(&p->p, ret);
	}
	put_poll_portal();
	return ret;
}
EXPORT_SYMBOL(bman_poll_slow);

/* Legacy wrapper */
void bman_poll(void)
{
	struct bman_portal *p = get_poll_portal();

#ifdef FSL_DPA_PORTAL_SHARE
#ifndef __rtems__
	if (unlikely(p->sharing_redirect))
		goto done;
#endif /* __rtems__ */
#endif
	if (!(p->slowpoll--)) {
		u32 is = bm_isr_status_read(&p->p) & ~p->irq_sources;
		u32 active = __poll_portal_slow(p, is);

		if (active)
			p->slowpoll = SLOW_POLL_BUSY;
		else
			p->slowpoll = SLOW_POLL_IDLE;
	}
#ifdef FSL_DPA_PORTAL_SHARE
#ifndef __rtems__
done:
#endif /* __rtems__ */
#endif
	put_poll_portal();
}
EXPORT_SYMBOL(bman_poll);

static const u32 zero_thresholds[4] = {0, 0, 0, 0};

struct bman_pool *bman_new_pool(const struct bman_pool_params *params)
{
	struct bman_pool *pool = NULL;
	u32 bpid;

	if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID) {
		if (bman_alloc_bpid(&bpid))
			return NULL;
	} else {
		if (params->bpid >= bman_pool_max)
			return NULL;
		bpid = params->bpid;
	}
#ifdef CONFIG_FSL_BMAN
	if (params->flags & BMAN_POOL_FLAG_THRESH) {
		if (bm_pool_set(bpid, params->thresholds))
			goto err;
	}
#else
	if (params->flags & BMAN_POOL_FLAG_THRESH)
		goto err;
#endif
	pool = kmalloc(sizeof(*pool), GFP_KERNEL);
	if (!pool)
		goto err;
	pool->sp = NULL;
	pool->sp_fill = 0;
	pool->params = *params;
#ifdef CONFIG_FSL_DPA_CHECKING
	atomic_set(&pool->in_use, 1);
#endif
	if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID)
		pool->params.bpid = bpid;
	if (params->flags & BMAN_POOL_FLAG_STOCKPILE) {
		pool->sp = kmalloc(sizeof(struct bm_buffer) * BMAN_STOCKPILE_SZ,
					GFP_KERNEL);
		if (!pool->sp)
			goto err;
	}
	if (pool->params.flags & BMAN_POOL_FLAG_DEPLETION) {
		struct bman_portal *p = get_affine_portal();

		if (!p->pools || !bman_depletion_get(&p->pools[0], bpid)) {
			pr_err("Depletion events disabled for bpid %d\n", bpid);
			goto err;
		}
		depletion_link(p, pool);
		put_affine_portal();
	}
	return pool;
err:
#ifdef CONFIG_FSL_BMAN
	if (params->flags & BMAN_POOL_FLAG_THRESH)
		bm_pool_set(bpid, zero_thresholds);
#endif
	if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID)
		bman_release_bpid(bpid);
	if (pool) {
		kfree(pool->sp);
		kfree(pool);
	}
	return NULL;
}
EXPORT_SYMBOL(bman_new_pool);

void bman_free_pool(struct bman_pool *pool)
{
#ifdef CONFIG_FSL_BMAN
	if (pool->params.flags & BMAN_POOL_FLAG_THRESH)
		bm_pool_set(pool->params.bpid, zero_thresholds);
#endif
	if (pool->params.flags & BMAN_POOL_FLAG_DEPLETION)
		depletion_unlink(pool);
	if (pool->params.flags & BMAN_POOL_FLAG_STOCKPILE) {
		if (pool->sp_fill)
			pr_err("Stockpile not flushed, has %u in bpid %u.\n",
				pool->sp_fill, pool->params.bpid);
		kfree(pool->sp);
		pool->sp = NULL;
		pool->params.flags ^= BMAN_POOL_FLAG_STOCKPILE;
	}
	if (pool->params.flags & BMAN_POOL_FLAG_DYNAMIC_BPID)
		bman_release_bpid(pool->params.bpid);
	kfree(pool);
}
EXPORT_SYMBOL(bman_free_pool);

const struct bman_pool_params *bman_get_params(const struct bman_pool *pool)
{
	return &pool->params;
}
EXPORT_SYMBOL(bman_get_params);

static noinline void update_rcr_ci(struct bman_portal *p, u8 avail)
{
	if (avail)
		bm_rcr_cce_prefetch(&p->p);
	else
		bm_rcr_cce_update(&p->p);
}

int bman_rcr_is_empty(void)
{
	__maybe_unused unsigned long irqflags;
	struct bman_portal *p = get_affine_portal();
	u8 avail;

	PORTAL_IRQ_LOCK(p, irqflags);
	update_rcr_ci(p, 0);
	avail = bm_rcr_get_fill(&p->p);
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
	return avail == 0;
}
EXPORT_SYMBOL(bman_rcr_is_empty);

static inline struct bm_rcr_entry *try_rel_start(struct bman_portal **p,
#ifdef FSL_DPA_CAN_WAIT
					__maybe_unused struct bman_pool *pool,
#endif
					__maybe_unused unsigned long *irqflags,
					__maybe_unused u32 flags)
{
	struct bm_rcr_entry *r;
	u8 avail;

	*p = get_affine_portal();
	PORTAL_IRQ_LOCK(*p, (*irqflags));
#ifdef FSL_DPA_CAN_WAIT_SYNC
	if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) &&
			(flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) {
		if ((*p)->rcri_owned) {
			PORTAL_IRQ_UNLOCK(*p, (*irqflags));
			put_affine_portal();
			return NULL;
		}
		(*p)->rcri_owned = pool;
	}
#endif
	avail = bm_rcr_get_avail(&(*p)->p);
	if (avail < 2)
		update_rcr_ci(*p, avail);
	r = bm_rcr_start(&(*p)->p);
	if (unlikely(!r)) {
#ifdef FSL_DPA_CAN_WAIT_SYNC
		if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) &&
				(flags & BMAN_RELEASE_FLAG_WAIT_SYNC)))
			(*p)->rcri_owned = NULL;
#endif
		PORTAL_IRQ_UNLOCK(*p, (*irqflags));
		put_affine_portal();
	}
	return r;
}

#ifdef FSL_DPA_CAN_WAIT
static noinline struct bm_rcr_entry *__wait_rel_start(struct bman_portal **p,
					struct bman_pool *pool,
					__maybe_unused unsigned long *irqflags,
					u32 flags)
{
	struct bm_rcr_entry *rcr = try_rel_start(p, pool, irqflags, flags);

	if (!rcr)
		bm_rcr_set_ithresh(&(*p)->p, 1);
	return rcr;
}

static noinline struct bm_rcr_entry *wait_rel_start(struct bman_portal **p,
					struct bman_pool *pool,
					__maybe_unused unsigned long *irqflags,
					u32 flags)
{
	struct bm_rcr_entry *rcr;
#ifndef FSL_DPA_CAN_WAIT_SYNC
	pool = NULL;
#endif
#ifndef __rtems__
	if (flags & BMAN_RELEASE_FLAG_WAIT_INT)
		wait_event_interruptible(affine_queue,
			(rcr = __wait_rel_start(p, pool, irqflags, flags)));
	else
#endif /* __rtems__ */
		wait_event(affine_queue,
			(rcr = __wait_rel_start(p, pool, irqflags, flags)));
	return rcr;
}
#endif

/* to facilitate better copying of bufs into the ring without either (a) copying
 * noise into the first byte (prematurely triggering the command), nor (b) being
 * very inefficient by copying small fields using read-modify-write */
struct overlay_bm_buffer {
	u32 first;
	u32 second;
};

static inline int __bman_release(struct bman_pool *pool,
			const struct bm_buffer *bufs, u8 num, u32 flags)
{
	struct bman_portal *p;
	struct bm_rcr_entry *r;
	struct overlay_bm_buffer *o_dest;
	struct overlay_bm_buffer *o_src = (struct overlay_bm_buffer *)&bufs[0];
	__maybe_unused unsigned long irqflags;
	u32 i = num - 1;

#ifdef FSL_DPA_CAN_WAIT
	if (flags & BMAN_RELEASE_FLAG_WAIT)
		r = wait_rel_start(&p, pool, &irqflags, flags);
	else
		r = try_rel_start(&p, pool, &irqflags, flags);
#else
	r = try_rel_start(&p, &irqflags, flags);
#endif
	if (!r)
		return -EBUSY;
	/* We can copy all but the first entry, as this can trigger badness
	 * with the valid-bit. Use the overlay to mask the verb byte. */
	o_dest = (struct overlay_bm_buffer *)&r->bufs[0];
	o_dest->first = (o_src->first & 0x0000ffff) |
		(((u32)pool->params.bpid << 16) & 0x00ff0000);
	o_dest->second = o_src->second;
	if (i)
		copy_words(&r->bufs[1], &bufs[1], i * sizeof(bufs[0]));
	bm_rcr_pvb_commit(&p->p, BM_RCR_VERB_CMD_BPID_SINGLE |
			(num & BM_RCR_VERB_BUFCOUNT_MASK));
#ifdef FSL_DPA_CAN_WAIT_SYNC
	/* if we wish to sync we need to set the threshold after h/w sees the
	 * new ring entry. As we're mixing cache-enabled and cache-inhibited
	 * accesses, this requires a heavy-weight sync. */
	if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) &&
			(flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) {
		hwsync();
		bm_rcr_set_ithresh(&p->p, 1);
	}
#endif
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
#ifdef FSL_DPA_CAN_WAIT_SYNC
	if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) &&
			(flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) {
#ifndef __rtems__
		if (flags & BMAN_RELEASE_FLAG_WAIT_INT)
			wait_event_interruptible(affine_queue,
					(p->rcri_owned != pool));
		else
#endif /* __rtems__ */
			wait_event(affine_queue, (p->rcri_owned != pool));
	}
#endif
	return 0;
}

int bman_release(struct bman_pool *pool, const struct bm_buffer *bufs, u8 num,
			u32 flags)
{
	int ret = 0;

#ifdef CONFIG_FSL_DPA_CHECKING
	if (!num || (num > 8))
		return -EINVAL;
	if (pool->params.flags & BMAN_POOL_FLAG_NO_RELEASE)
		return -EINVAL;
#endif
	/* Without stockpile, this API is a pass-through to the h/w operation */
	if (!(pool->params.flags & BMAN_POOL_FLAG_STOCKPILE))
		return __bman_release(pool, bufs, num, flags);
#ifdef CONFIG_FSL_DPA_CHECKING
	if (!atomic_dec_and_test(&pool->in_use)) {
		pr_crit("Parallel attempts to enter bman_released() detected.");
		panic("only one instance of bman_released/acquired allowed");
	}
#endif
	/* This needs some explanation. Adding the given buffers may take the
	 * stockpile over the threshold, but in fact the stockpile may already
	 * *be* over the threshold if a previous release-to-hw attempt had
	 * failed. So we have 3 cases to cover;
	 *   1. we add to the stockpile and don't hit the threshold,
	 *   2. we add to the stockpile, hit the threshold and release-to-hw,
	 *   3. we have to release-to-hw before adding to the stockpile
	 *	(not enough room in the stockpile for case 2).
	 * Our constraints on thresholds guarantee that in case 3, there must be
	 * at least 8 bufs already in the stockpile, so all release-to-hw ops
	 * are for 8 bufs. Despite all this, the API must indicate whether the
	 * given buffers were taken off the caller's hands, irrespective of
	 * whether a release-to-hw was attempted. */
	while (num) {
		/* Add buffers to stockpile if they fit */
		if ((pool->sp_fill + num) < BMAN_STOCKPILE_SZ) {
			copy_words(pool->sp + pool->sp_fill, bufs,
				sizeof(struct bm_buffer) * num);
			pool->sp_fill += num;
			num = 0; /* --> will return success no matter what */
		}
		/* Do hw op if hitting the high-water threshold */
		if ((pool->sp_fill + num) >= BMAN_STOCKPILE_HIGH) {
			ret = __bman_release(pool,
				pool->sp + (pool->sp_fill - 8), 8, flags);
			if (ret) {
				ret = (num ? ret : 0);
				goto release_done;
			}
			pool->sp_fill -= 8;
		}
	}
release_done:
#ifdef CONFIG_FSL_DPA_CHECKING
	atomic_inc(&pool->in_use);
#endif
	return ret;
}
EXPORT_SYMBOL(bman_release);

static inline int __bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs,
					u8 num)
{
	struct bman_portal *p = get_affine_portal();
	struct bm_mc_command *mcc;
	struct bm_mc_result *mcr;
	__maybe_unused unsigned long irqflags;
	int ret;

	PORTAL_IRQ_LOCK(p, irqflags);
	mcc = bm_mc_start(&p->p);
	mcc->acquire.bpid = pool->params.bpid;
	bm_mc_commit(&p->p, BM_MCC_VERB_CMD_ACQUIRE |
			(num & BM_MCC_VERB_ACQUIRE_BUFCOUNT));
	while (!(mcr = bm_mc_result(&p->p)))
		cpu_relax();
	ret = mcr->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT;
	if (bufs)
		copy_words(&bufs[0], &mcr->acquire.bufs[0],
				num * sizeof(bufs[0]));
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
	if (ret != num)
		ret = -ENOMEM;
	return ret;
}

int bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, u8 num,
			u32 flags)
{
	int ret = 0;

#ifdef CONFIG_FSL_DPA_CHECKING
	if (!num || (num > 8))
		return -EINVAL;
	if (pool->params.flags & BMAN_POOL_FLAG_ONLY_RELEASE)
		return -EINVAL;
#endif
	/* Without stockpile, this API is a pass-through to the h/w operation */
	if (!(pool->params.flags & BMAN_POOL_FLAG_STOCKPILE))
		return __bman_acquire(pool, bufs, num);
#ifdef CONFIG_FSL_DPA_CHECKING
	if (!atomic_dec_and_test(&pool->in_use)) {
		pr_crit("Parallel attempts to enter bman_acquire() detected.");
		panic("only one instance of bman_released/acquired allowed");
	}
#endif
	/* Only need a h/w op if we'll hit the low-water thresh */
	if (!(flags & BMAN_ACQUIRE_FLAG_STOCKPILE) &&
			(pool->sp_fill <= (BMAN_STOCKPILE_LOW + num))) {
		/* refill stockpile with max amount, but if max amount
		 * isn't available, try amount the user wants */
		int bufcount = 8;

		ret = __bman_acquire(pool, pool->sp + pool->sp_fill, bufcount);
		if (ret < 0 && bufcount != num) {
			bufcount = num;
			/* Maybe buffer pool has less than 8 */
			ret = __bman_acquire(pool, pool->sp + pool->sp_fill,
						bufcount);
		}
		if (ret < 0)
			goto hw_starved;
		DPA_ASSERT(ret == bufcount);
		pool->sp_fill += bufcount;
	} else {
hw_starved:
		if (pool->sp_fill < num) {
			ret = -ENOMEM;
			goto acquire_done;
		}
	}
	copy_words(bufs, pool->sp + (pool->sp_fill - num),
		sizeof(struct bm_buffer) * num);
	pool->sp_fill -= num;
	ret = num;
acquire_done:
#ifdef CONFIG_FSL_DPA_CHECKING
	atomic_inc(&pool->in_use);
#endif
	return ret;
}
EXPORT_SYMBOL(bman_acquire);

int bman_flush_stockpile(struct bman_pool *pool, u32 flags)
{
	u8 num;
	int ret;

	while (pool->sp_fill) {
		num = ((pool->sp_fill > 8) ? 8 : pool->sp_fill);
		ret = __bman_release(pool, pool->sp + (pool->sp_fill - num),
				     num, flags);
		if (ret)
			return ret;
		pool->sp_fill -= num;
	}
	return 0;
}
EXPORT_SYMBOL(bman_flush_stockpile);

int bman_query_pools(struct bm_pool_state *state)
{
	struct bman_portal *p = get_affine_portal();
	struct bm_mc_result *mcr;
	__maybe_unused unsigned long irqflags;

	PORTAL_IRQ_LOCK(p, irqflags);
	bm_mc_start(&p->p);
	bm_mc_commit(&p->p, BM_MCC_VERB_CMD_QUERY);
	while (!(mcr = bm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & BM_MCR_VERB_CMD_MASK) == BM_MCR_VERB_CMD_QUERY);
	*state = mcr->query;
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
	return 0;
}
EXPORT_SYMBOL(bman_query_pools);

#ifdef CONFIG_FSL_BMAN
u32 bman_query_free_buffers(struct bman_pool *pool)
{
	return bm_pool_free_buffers(pool->params.bpid);
}
EXPORT_SYMBOL(bman_query_free_buffers);

int bman_update_pool_thresholds(struct bman_pool *pool, const u32 *thresholds)
{
	u32 bpid;

	bpid = bman_get_params(pool)->bpid;

	return bm_pool_set(bpid, thresholds);
}
EXPORT_SYMBOL(bman_update_pool_thresholds);
#endif

int bman_shutdown_pool(u32 bpid)
{
	struct bman_portal *p = get_affine_portal();
	__maybe_unused unsigned long irqflags;
	int ret;

	PORTAL_IRQ_LOCK(p, irqflags);
	ret = bm_shutdown_pool(&p->p, bpid);
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
	return ret;
}
EXPORT_SYMBOL(bman_shutdown_pool);

const struct bm_portal_config *
bman_get_bm_portal_config(const struct bman_portal *portal)
{
#ifndef __rtems__
	return portal->sharing_redirect ? NULL : portal->config;
#else /* __rtems__ */
	return portal->config;
#endif /* __rtems__ */
}