diff options
Diffstat (limited to 'linux/drivers/soc/fsl/qbman/qman_api.c')
| -rw-r--r-- | linux/drivers/soc/fsl/qbman/qman_api.c | 3026 |
1 files changed, 3026 insertions, 0 deletions
diff --git a/linux/drivers/soc/fsl/qbman/qman_api.c b/linux/drivers/soc/fsl/qbman/qman_api.c new file mode 100644 index 00000000..e838d08f --- /dev/null +++ b/linux/drivers/soc/fsl/qbman/qman_api.c @@ -0,0 +1,3026 @@ +#include <machine/rtems-bsd-kernel-space.h> + +#include <rtems/bsd/local/opt_dpaa.h> + +/* Copyright 2008 - 2015 Freescale Semiconductor, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Freescale Semiconductor nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "qman.h" + +/* Compilation constants */ +#define DQRR_MAXFILL 15 +#define EQCR_ITHRESH 4 /* if EQCR congests, interrupt threshold */ +#define IRQNAME "QMan portal %d" +#define MAX_IRQNAME 16 /* big enough for "QMan portal %d" */ +#define QMAN_POLL_LIMIT 32 +#define QMAN_PIRQ_DQRR_ITHRESH 12 +#define QMAN_PIRQ_MR_ITHRESH 4 +#define QMAN_PIRQ_IPERIOD 100 +#define FSL_DPA_PORTAL_SHARE 1 /* Allow portals to be shared */ +/* Divide 'n' by 'd', rounding down if 'r' is negative, rounding up if it's + * positive, and rounding to the closest value if it's zero. NB, this macro + * implicitly upgrades parameters to unsigned 64-bit, so feed it with types + * that are compatible with this. NB, these arguments should not be expressions + * unless it is safe for them to be evaluated multiple times. Eg. do not pass + * in "some_value++" as a parameter to the macro! */ +#define ROUNDING(n, d, r) \ + (((r) < 0) ? div64_u64((n), (d)) : \ + (((r) > 0) ? div64_u64(((n) + (d) - 1), (d)) : \ + div64_u64(((n) + ((d) / 2)), (d)))) + +/* Lock/unlock frame queues, subject to the "LOCKED" flag. This is about + * inter-processor locking only. Note, FQLOCK() is always called either under a + * local_irq_save() or from interrupt context - hence there's no need for irq + * protection (and indeed, attempting to nest irq-protection doesn't work, as + * the "irq en/disable" machinery isn't recursive...). */ +#define FQLOCK(fq) \ + do { \ + struct qman_fq *__fq478 = (fq); \ + if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \ + spin_lock(&__fq478->fqlock); \ + } while (0) +#define FQUNLOCK(fq) \ + do { \ + struct qman_fq *__fq478 = (fq); \ + if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \ + spin_unlock(&__fq478->fqlock); \ + } while (0) + +static inline void fq_set(struct qman_fq *fq, u32 mask) +{ + set_bits(mask, &fq->flags); +} +static inline void fq_clear(struct qman_fq *fq, u32 mask) +{ + clear_bits(mask, &fq->flags); +} +static inline int fq_isset(struct qman_fq *fq, u32 mask) +{ + return fq->flags & mask; +} +static inline int fq_isclear(struct qman_fq *fq, u32 mask) +{ + return !(fq->flags & mask); +} + +struct qman_portal { + struct qm_portal p; + unsigned long bits; /* PORTAL_BITS_*** - dynamic, strictly internal */ + unsigned long irq_sources; + u32 use_eqcr_ci_stashing; + u32 slowpoll; /* only used when interrupts are off */ + struct qman_fq *vdqcr_owned; /* only 1 volatile dequeue at a time */ +#ifdef FSL_DPA_CAN_WAIT_SYNC + struct qman_fq *eqci_owned; /* only 1 enqueue WAIT_SYNC at a time */ +#endif +#ifdef FSL_DPA_PORTAL_SHARE + raw_spinlock_t sharing_lock; /* only used if is_shared */ +#ifndef __rtems__ + int is_shared; + struct qman_portal *sharing_redirect; +#endif /* __rtems__ */ +#endif + u32 sdqcr; + int dqrr_disable_ref; + /* A portal-specific handler for DCP ERNs. If this is NULL, the global + * handler is called instead. */ + qman_cb_dc_ern cb_dc_ern; + /* When the cpu-affine portal is activated, this is non-NULL */ + const struct qm_portal_config *config; +#ifndef __rtems__ + /* This is needed for providing a non-NULL device to dma_map_***() */ + struct platform_device *pdev; +#endif /* __rtems__ */ + struct dpa_rbtree retire_table; + char irqname[MAX_IRQNAME]; + /* 2-element array. cgrs[0] is mask, cgrs[1] is snapshot. */ + struct qman_cgrs *cgrs; + /* linked-list of CSCN handlers. */ + struct list_head cgr_cbs; + /* list lock */ + spinlock_t cgr_lock; + /* track if memory was allocated by the driver */ + u8 alloced; +}; + +#ifdef FSL_DPA_PORTAL_SHARE +#ifndef __rtems__ +#define PORTAL_IRQ_LOCK(p, irqflags) \ + do { \ + if ((p)->is_shared) \ + raw_spin_lock_irqsave(&(p)->sharing_lock, irqflags); \ + else \ + local_irq_save(irqflags); \ + } while (0) +#define PORTAL_IRQ_UNLOCK(p, irqflags) \ + do { \ + if ((p)->is_shared) \ + raw_spin_unlock_irqrestore(&(p)->sharing_lock, \ + irqflags); \ + else \ + local_irq_restore(irqflags); \ + } while (0) +#else /* __rtems__ */ +#define PORTAL_IRQ_LOCK(p, irqflags) \ + raw_spin_lock_irqsave(&(p)->sharing_lock, irqflags) +#define PORTAL_IRQ_UNLOCK(p, irqflags) \ + raw_spin_unlock_irqrestore(&(p)->sharing_lock, irqflags) +#endif /* __rtems__ */ +#else +#define PORTAL_IRQ_LOCK(p, irqflags) local_irq_save(irqflags) +#define PORTAL_IRQ_UNLOCK(p, irqflags) local_irq_restore(irqflags) +#endif + +/* Global handler for DCP ERNs. Used when the portal receiving the message does + * not have a portal-specific handler. */ +static qman_cb_dc_ern cb_dc_ern; + +#ifndef __rtems__ +static cpumask_t affine_mask; +static DEFINE_SPINLOCK(affine_mask_lock); +static u16 affine_channels[NR_CPUS]; +#endif /* __rtems__ */ +static DEFINE_PER_CPU(struct qman_portal, qman_affine_portal); +#ifndef __rtems__ +void *affine_portals[NR_CPUS]; +#endif /* __rtems__ */ + +/* "raw" gets the cpu-local struct whether it's a redirect or not. */ +static inline struct qman_portal *get_raw_affine_portal(void) +{ + return &get_cpu_var(qman_affine_portal); +} +/* For ops that can redirect, this obtains the portal to use */ +#ifdef FSL_DPA_PORTAL_SHARE +static inline struct qman_portal *get_affine_portal(void) +{ + struct qman_portal *p = get_raw_affine_portal(); + +#ifndef __rtems__ + if (p->sharing_redirect) + return p->sharing_redirect; +#endif /* __rtems__ */ + return p; +} +#else +#define get_affine_portal() get_raw_affine_portal() +#endif +/* For every "get", there must be a "put" */ +static inline void put_affine_portal(void) +{ + put_cpu_var(qman_affine_portal); +} +/* Exception: poll functions assume the caller is cpu-affine and in no risk of + * re-entrance, which are the two reasons we usually use the get/put_cpu_var() + * semantic - ie. to disable pre-emption. Some use-cases expect the execution + * context to remain as non-atomic during poll-triggered callbacks as it was + * when the poll API was first called (eg. NAPI), so we go out of our way in + * this case to not disable pre-emption. */ +static inline struct qman_portal *get_poll_portal(void) +{ + return this_cpu_ptr(&qman_affine_portal); +} +#define put_poll_portal() + +/* This gives a FQID->FQ lookup to cover the fact that we can't directly demux + * retirement notifications (the fact they are sometimes h/w-consumed means that + * contextB isn't always a s/w demux - and as we can't know which case it is + * when looking at the notification, we have to use the slow lookup for all of + * them). NB, it's possible to have multiple FQ objects refer to the same FQID + * (though at most one of them should be the consumer), so this table isn't for + * all FQs - FQs are added when retirement commands are issued, and removed when + * they complete, which also massively reduces the size of this table. */ +IMPLEMENT_DPA_RBTREE(fqtree, struct qman_fq, node, fqid); + +/* This is what everything can wait on, even if it migrates to a different cpu + * to the one whose affine portal it is waiting on. */ +static DECLARE_WAIT_QUEUE_HEAD(affine_queue); + +static inline int table_push_fq(struct qman_portal *p, struct qman_fq *fq) +{ + int ret = fqtree_push(&p->retire_table, fq); + + if (ret) + pr_err("ERROR: double FQ-retirement %d\n", fq->fqid); + return ret; +} + +static inline void table_del_fq(struct qman_portal *p, struct qman_fq *fq) +{ + fqtree_del(&p->retire_table, fq); +} + +static inline struct qman_fq *table_find_fq(struct qman_portal *p, u32 fqid) +{ + return fqtree_find(&p->retire_table, fqid); +} + +#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP +static void **qman_fq_lookup_table; +static size_t qman_fq_lookup_table_size; + +int qman_setup_fq_lookup_table(size_t num_entries) +{ + num_entries++; + /* Allocate 1 more entry since the first entry is not used */ + qman_fq_lookup_table = vzalloc((num_entries * sizeof(void *))); + if (!qman_fq_lookup_table) + return -ENOMEM; + qman_fq_lookup_table_size = num_entries; + pr_info("Allocated lookup table at %p, entry count %lu\n", + qman_fq_lookup_table, (unsigned long)qman_fq_lookup_table_size); + return 0; +} + +/* global structure that maintains fq object mapping */ +static DEFINE_SPINLOCK(fq_hash_table_lock); + +static int find_empty_fq_table_entry(u32 *entry, struct qman_fq *fq) +{ + u32 i; + + spin_lock(&fq_hash_table_lock); + /* Can't use index zero because this has special meaning + * in context_b field. */ + for (i = 1; i < qman_fq_lookup_table_size; i++) { + if (qman_fq_lookup_table[i] == NULL) { + *entry = i; + qman_fq_lookup_table[i] = fq; + spin_unlock(&fq_hash_table_lock); + return 0; + } + } + spin_unlock(&fq_hash_table_lock); + return -ENOMEM; +} + +static void clear_fq_table_entry(u32 entry) +{ + spin_lock(&fq_hash_table_lock); + BUG_ON(entry >= qman_fq_lookup_table_size); + qman_fq_lookup_table[entry] = NULL; + spin_unlock(&fq_hash_table_lock); +} + +static inline struct qman_fq *get_fq_table_entry(u32 entry) +{ + BUG_ON(entry >= qman_fq_lookup_table_size); + return qman_fq_lookup_table[entry]; +} +#endif + +/* In the case that slow- and fast-path handling are both done by qman_poll() + * (ie. because there is no interrupt handling), we ought to balance how often + * we do the fast-path poll versus the slow-path poll. We'll use two decrementer + * sources, so we call the fast poll 'n' times before calling the slow poll + * once. The idle decrementer constant is used when the last slow-poll detected + * no work to do, and the busy decrementer constant when the last slow-poll had + * work to do. */ +#define SLOW_POLL_IDLE 1000 +#define SLOW_POLL_BUSY 10 +static u32 __poll_portal_slow(struct qman_portal *p, u32 is); +static inline unsigned int __poll_portal_fast(struct qman_portal *p, + unsigned int poll_limit); + +/* Portal interrupt handler */ +static irqreturn_t portal_isr(__always_unused int irq, void *ptr) +{ + struct qman_portal *p = ptr; + /* + * The CSCI source is cleared inside __poll_portal_slow(), because + * it could race against a Query Congestion State command also given + * as part of the handling of this interrupt source. We mustn't + * clear it a second time in this top-level function. + */ + u32 clear = QM_DQAVAIL_MASK; + u32 is = qm_isr_status_read(&p->p) & p->irq_sources; + /* DQRR-handling if it's interrupt-driven */ + if (is & QM_PIRQ_DQRI) { + clear |= QM_PIRQ_DQRI; + __poll_portal_fast(p, QMAN_POLL_LIMIT); + } + /* Handling of anything else that's interrupt-driven */ + clear |= __poll_portal_slow(p, is); + qm_isr_status_clear(&p->p, clear); + return IRQ_HANDLED; +} + +/* This inner version is used privately by qman_create_affine_portal(), as well + * as by the exported qman_stop_dequeues(). */ +static inline void qman_stop_dequeues_ex(struct qman_portal *p) +{ + unsigned long irqflags __maybe_unused; + PORTAL_IRQ_LOCK(p, irqflags); + if (!(p->dqrr_disable_ref++)) + qm_dqrr_set_maxfill(&p->p, 0); + PORTAL_IRQ_UNLOCK(p, irqflags); +} + +static int drain_mr_fqrni(struct qm_portal *p) +{ + const struct qm_mr_entry *msg; +loop: + msg = qm_mr_current(p); + if (!msg) { + /* if MR was full and h/w had other FQRNI entries to produce, we + * need to allow it time to produce those entries once the + * existing entries are consumed. A worst-case situation + * (fully-loaded system) means h/w sequencers may have to do 3-4 + * other things before servicing the portal's MR pump, each of + * which (if slow) may take ~50 qman cycles (which is ~200 + * processor cycles). So rounding up and then multiplying this + * worst-case estimate by a factor of 10, just to be + * ultra-paranoid, goes as high as 10,000 cycles. NB, we consume + * one entry at a time, so h/w has an opportunity to produce new + * entries well before the ring has been fully consumed, so + * we're being *really* paranoid here. */ + u64 now, then = mfatb(); + + do { + now = mfatb(); + } while ((then + 10000) > now); + msg = qm_mr_current(p); + if (!msg) + return 0; + } + if ((msg->verb & QM_MR_VERB_TYPE_MASK) != QM_MR_VERB_FQRNI) { + /* We aren't draining anything but FQRNIs */ + pr_err("Found verb 0x%x in MR\n", msg->verb); + return -1; + } + qm_mr_next(p); + qm_mr_cci_consume(p, 1); + goto loop; +} + +struct qman_portal *qman_create_portal( + struct qman_portal *portal, + const struct qm_portal_config *config, + const struct qman_cgrs *cgrs) +{ + struct qm_portal *__p; +#ifndef __rtems__ + char buf[16]; +#endif /* __rtems__ */ + int ret; + u32 isdr; + + if (!portal) { + portal = kmalloc(sizeof(*portal), GFP_KERNEL); + if (!portal) + return portal; + portal->alloced = 1; + } else + portal->alloced = 0; + + __p = &portal->p; + +#ifndef __rtems__ + portal->use_eqcr_ci_stashing = ((qman_ip_rev >= QMAN_REV30) ? + 1 : 0); +#else /* __rtems__ */ + portal->use_eqcr_ci_stashing = 0; +#endif /* __rtems__ */ + + /* prep the low-level portal struct with the mapped addresses from the + * config, everything that follows depends on it and "config" is more + * for (de)reference... */ + __p->addr.addr_ce = config->addr_virt[DPA_PORTAL_CE]; + __p->addr.addr_ci = config->addr_virt[DPA_PORTAL_CI]; + /* + * If CI-stashing is used, the current defaults use a threshold of 3, + * and stash with high-than-DQRR priority. + */ + if (qm_eqcr_init(__p, qm_eqcr_pvb, + portal->use_eqcr_ci_stashing ? 3 : 0, 1)) { + pr_err("EQCR initialisation failed\n"); + goto fail_eqcr; + } + if (qm_dqrr_init(__p, config, qm_dqrr_dpush, qm_dqrr_pvb, + qm_dqrr_cdc, DQRR_MAXFILL)) { + pr_err("DQRR initialisation failed\n"); + goto fail_dqrr; + } + if (qm_mr_init(__p, qm_mr_pvb, qm_mr_cci)) { + pr_err("MR initialisation failed\n"); + goto fail_mr; + } + if (qm_mc_init(__p)) { + pr_err("MC initialisation failed\n"); + goto fail_mc; + } + if (qm_isr_init(__p)) { + pr_err("ISR initialisation failed\n"); + goto fail_isr; + } + /* static interrupt-gating controls */ + qm_dqrr_set_ithresh(__p, QMAN_PIRQ_DQRR_ITHRESH); + qm_mr_set_ithresh(__p, QMAN_PIRQ_MR_ITHRESH); + qm_isr_set_iperiod(__p, QMAN_PIRQ_IPERIOD); + portal->cgrs = kmalloc(2 * sizeof(*cgrs), GFP_KERNEL); + if (!portal->cgrs) + goto fail_cgrs; + /* initial snapshot is no-depletion */ + qman_cgrs_init(&portal->cgrs[1]); + if (cgrs) + portal->cgrs[0] = *cgrs; + else + /* if the given mask is NULL, assume all CGRs can be seen */ + qman_cgrs_fill(&portal->cgrs[0]); + INIT_LIST_HEAD(&portal->cgr_cbs); + spin_lock_init(&portal->cgr_lock); + portal->bits = 0; + portal->slowpoll = 0; +#ifdef FSL_DPA_CAN_WAIT_SYNC + portal->eqci_owned = NULL; +#endif +#ifdef FSL_DPA_PORTAL_SHARE + raw_spin_lock_init(&portal->sharing_lock); +#ifndef __rtems__ + portal->is_shared = config->public_cfg.is_shared; + portal->sharing_redirect = NULL; +#endif /* __rtems__ */ +#endif + portal->sdqcr = QM_SDQCR_SOURCE_CHANNELS | QM_SDQCR_COUNT_UPTO3 | + QM_SDQCR_DEDICATED_PRECEDENCE | QM_SDQCR_TYPE_PRIO_QOS | + QM_SDQCR_TOKEN_SET(0xab) | QM_SDQCR_CHANNELS_DEDICATED; + portal->dqrr_disable_ref = 0; + portal->cb_dc_ern = NULL; +#ifndef __rtems__ + sprintf(buf, "qportal-%d", config->public_cfg.channel); + portal->pdev = platform_device_alloc(buf, -1); + if (!portal->pdev) + goto fail_devalloc; + if (dma_set_mask(&portal->pdev->dev, DMA_BIT_MASK(40))) + goto fail_devadd; + ret = platform_device_add(portal->pdev); + if (ret) + goto fail_devadd; +#endif /* __rtems__ */ + dpa_rbtree_init(&portal->retire_table); + isdr = 0xffffffff; + qm_isr_disable_write(__p, isdr); + portal->irq_sources = 0; + qm_isr_enable_write(__p, portal->irq_sources); + qm_isr_status_clear(__p, 0xffffffff); + snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, config->public_cfg.cpu); + if (request_irq(config->public_cfg.irq, portal_isr, 0, portal->irqname, + portal)) { + pr_err("request_irq() failed\n"); + goto fail_irq; + } +#ifndef __rtems__ + if ((config->public_cfg.cpu != -1) && + irq_can_set_affinity(config->public_cfg.irq) && + irq_set_affinity(config->public_cfg.irq, + cpumask_of(config->public_cfg.cpu))) { + pr_err("irq_set_affinity() failed\n"); + goto fail_affinity; + } +#endif /* __rtems__ */ + + /* Need EQCR to be empty before continuing */ + isdr ^= QM_PIRQ_EQCI; + qm_isr_disable_write(__p, isdr); + ret = qm_eqcr_get_fill(__p); + if (ret) { + pr_err("EQCR unclean\n"); + goto fail_eqcr_empty; + } + isdr ^= (QM_PIRQ_DQRI | QM_PIRQ_MRI); + qm_isr_disable_write(__p, isdr); + if (qm_dqrr_current(__p) != NULL) { + pr_err("DQRR unclean\n"); + qm_dqrr_cdc_consume_n(__p, 0xffff); + } + if (qm_mr_current(__p) != NULL) { + /* special handling, drain just in case it's a few FQRNIs */ + if (drain_mr_fqrni(__p)) { + const struct qm_mr_entry *e = qm_mr_current(__p); + + pr_err("MR unclean, MR VERB 0x%x, rc 0x%x\n, addr 0x%x", + e->verb, e->ern.rc, e->ern.fd.addr_lo); + goto fail_dqrr_mr_empty; + } + } + /* Success */ + portal->config = config; + qm_isr_disable_write(__p, 0); + qm_isr_uninhibit(__p); + /* Write a sane SDQCR */ + qm_dqrr_sdqcr_set(__p, portal->sdqcr); + return portal; +fail_dqrr_mr_empty: +fail_eqcr_empty: +#ifndef __rtems__ +fail_affinity: +#endif /* __rtems__ */ + free_irq(config->public_cfg.irq, portal); +fail_irq: +#ifndef __rtems__ + platform_device_del(portal->pdev); +fail_devadd: + platform_device_put(portal->pdev); +fail_devalloc: +#endif /* __rtems__ */ + kfree(portal->cgrs); +fail_cgrs: + qm_isr_finish(__p); +fail_isr: + qm_mc_finish(__p); +fail_mc: + qm_mr_finish(__p); +fail_mr: + qm_dqrr_finish(__p); +fail_dqrr: + qm_eqcr_finish(__p); +fail_eqcr: + return NULL; +} + +struct qman_portal *qman_create_affine_portal( + const struct qm_portal_config *config, + const struct qman_cgrs *cgrs) +{ + struct qman_portal *res; + struct qman_portal *portal; + + portal = &per_cpu(qman_affine_portal, config->public_cfg.cpu); + res = qman_create_portal(portal, config, cgrs); + if (res) { +#ifndef __rtems__ + spin_lock(&affine_mask_lock); + cpumask_set_cpu(config->public_cfg.cpu, &affine_mask); + affine_channels[config->public_cfg.cpu] = + config->public_cfg.channel; + affine_portals[config->public_cfg.cpu] = portal; + spin_unlock(&affine_mask_lock); +#endif /* __rtems__ */ + } + return res; +} + +#ifndef __rtems__ +/* These checks are BUG_ON()s because the driver is already supposed to avoid + * these cases. */ +struct qman_portal *qman_create_affine_slave(struct qman_portal *redirect, + int cpu) +{ +#ifdef FSL_DPA_PORTAL_SHARE + struct qman_portal *p = &per_cpu(qman_affine_portal, cpu); + + /* Check that we don't already have our own portal */ + BUG_ON(p->config); + /* Check that we aren't already slaving to another portal */ + BUG_ON(p->is_shared); + /* Check that 'redirect' is prepared to have us */ + BUG_ON(!redirect->config->public_cfg.is_shared); + /* These are the only elements to initialise when redirecting */ + p->irq_sources = 0; + p->sharing_redirect = redirect; + affine_portals[cpu] = p; + return p; +#else + BUG(); + return NULL; +#endif +} +#endif /* __rtems__ */ + +void qman_destroy_portal(struct qman_portal *qm) +{ + const struct qm_portal_config *pcfg; + + /* Stop dequeues on the portal */ + qm_dqrr_sdqcr_set(&qm->p, 0); + + /* NB we do this to "quiesce" EQCR. If we add enqueue-completions or + * something related to QM_PIRQ_EQCI, this may need fixing. + * Also, due to the prefetching model used for CI updates in the enqueue + * path, this update will only invalidate the CI cacheline *after* + * working on it, so we need to call this twice to ensure a full update + * irrespective of where the enqueue processing was at when the teardown + * began. */ + qm_eqcr_cce_update(&qm->p); + qm_eqcr_cce_update(&qm->p); + pcfg = qm->config; + + free_irq(pcfg->public_cfg.irq, qm); + + kfree(qm->cgrs); + qm_isr_finish(&qm->p); + qm_mc_finish(&qm->p); + qm_mr_finish(&qm->p); + qm_dqrr_finish(&qm->p); + qm_eqcr_finish(&qm->p); + +#ifndef __rtems__ + platform_device_del(qm->pdev); + platform_device_put(qm->pdev); +#endif /* __rtems__ */ + + qm->config = NULL; + if (qm->alloced) + kfree(qm); +} + +const struct qm_portal_config *qman_destroy_affine_portal(void) +{ + /* We don't want to redirect if we're a slave, use "raw" */ + struct qman_portal *qm = get_raw_affine_portal(); + const struct qm_portal_config *pcfg; +#ifndef __rtems__ + int cpu; +#endif /* __rtems__ */ + +#ifdef FSL_DPA_PORTAL_SHARE +#ifndef __rtems__ + if (qm->sharing_redirect) { + qm->sharing_redirect = NULL; + put_affine_portal(); + return NULL; + } + qm->is_shared = 0; +#endif /* __rtems__ */ +#endif + pcfg = qm->config; +#ifndef __rtems__ + cpu = pcfg->public_cfg.cpu; +#endif /* __rtems__ */ + + qman_destroy_portal(qm); + +#ifndef __rtems__ + spin_lock(&affine_mask_lock); + cpumask_clear_cpu(cpu, &affine_mask); + spin_unlock(&affine_mask_lock); +#endif /* __rtems__ */ + put_affine_portal(); + return pcfg; +} + +const struct qman_portal_config *qman_p_get_portal_config(struct qman_portal *p) +{ + return &p->config->public_cfg; +} +EXPORT_SYMBOL(qman_p_get_portal_config); + +const struct qman_portal_config *qman_get_portal_config(void) +{ + struct qman_portal *p = get_affine_portal(); + const struct qman_portal_config *ret = qman_p_get_portal_config(p); + + put_affine_portal(); + return ret; +} +EXPORT_SYMBOL(qman_get_portal_config); + +/* Inline helper to reduce nesting in __poll_portal_slow() */ +static inline void fq_state_change(struct qman_portal *p, struct qman_fq *fq, + const struct qm_mr_entry *msg, u8 verb) +{ + FQLOCK(fq); + switch (verb) { + case QM_MR_VERB_FQRL: + DPA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_ORL)); + fq_clear(fq, QMAN_FQ_STATE_ORL); + table_del_fq(p, fq); + break; + case QM_MR_VERB_FQRN: + DPA_ASSERT((fq->state == qman_fq_state_parked) || + (fq->state == qman_fq_state_sched)); + DPA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_CHANGING)); + fq_clear(fq, QMAN_FQ_STATE_CHANGING); + if (msg->fq.fqs & QM_MR_FQS_NOTEMPTY) + fq_set(fq, QMAN_FQ_STATE_NE); + if (msg->fq.fqs & QM_MR_FQS_ORLPRESENT) + fq_set(fq, QMAN_FQ_STATE_ORL); + else + table_del_fq(p, fq); + fq->state = qman_fq_state_retired; + break; + case QM_MR_VERB_FQPN: + DPA_ASSERT(fq->state == qman_fq_state_sched); + DPA_ASSERT(fq_isclear(fq, QMAN_FQ_STATE_CHANGING)); + fq->state = qman_fq_state_parked; + } + FQUNLOCK(fq); +} + +static u32 __poll_portal_slow(struct qman_portal *p, u32 is) +{ + const struct qm_mr_entry *msg; + + if (is & QM_PIRQ_CSCI) { + struct qman_cgrs rr, c; + struct qm_mc_result *mcr; + struct qman_cgr *cgr; + unsigned long irqflags __maybe_unused; + + spin_lock_irqsave(&p->cgr_lock, irqflags); + /* + * The CSCI bit must be cleared _before_ issuing the + * Query Congestion State command, to ensure that a long + * CGR State Change callback cannot miss an intervening + * state change. + */ + qm_isr_status_clear(&p->p, QM_PIRQ_CSCI); + qm_mc_start(&p->p); + qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + /* mask out the ones I'm not interested in */ + qman_cgrs_and(&rr, (const struct qman_cgrs *) + &mcr->querycongestion.state, &p->cgrs[0]); + /* check previous snapshot for delta, enter/exit congestion */ + qman_cgrs_xor(&c, &rr, &p->cgrs[1]); + /* update snapshot */ + qman_cgrs_cp(&p->cgrs[1], &rr); + /* Invoke callback */ + list_for_each_entry(cgr, &p->cgr_cbs, node) + if (cgr->cb && qman_cgrs_get(&c, cgr->cgrid)) + cgr->cb(p, cgr, qman_cgrs_get(&rr, cgr->cgrid)); + spin_unlock_irqrestore(&p->cgr_lock, irqflags); + } + +#ifdef FSL_DPA_CAN_WAIT_SYNC + if (is & QM_PIRQ_EQCI) { + unsigned long irqflags; + + PORTAL_IRQ_LOCK(p, irqflags); + p->eqci_owned = NULL; + PORTAL_IRQ_UNLOCK(p, irqflags); + wake_up(&affine_queue); + } +#endif + + if (is & QM_PIRQ_EQRI) { + unsigned long irqflags __maybe_unused; + + PORTAL_IRQ_LOCK(p, irqflags); + qm_eqcr_cce_update(&p->p); + qm_eqcr_set_ithresh(&p->p, 0); + PORTAL_IRQ_UNLOCK(p, irqflags); + wake_up(&affine_queue); + } + + if (is & QM_PIRQ_MRI) { + struct qman_fq *fq; + u8 verb, num = 0; +mr_loop: + qm_mr_pvb_update(&p->p); + msg = qm_mr_current(&p->p); + if (!msg) + goto mr_done; + verb = msg->verb & QM_MR_VERB_TYPE_MASK; + /* The message is a software ERN iff the 0x20 bit is set */ + if (verb & 0x20) { + switch (verb) { + case QM_MR_VERB_FQRNI: + /* nada, we drop FQRNIs on the floor */ + break; + case QM_MR_VERB_FQRN: + case QM_MR_VERB_FQRL: + /* Lookup in the retirement table */ + fq = table_find_fq(p, msg->fq.fqid); + BUG_ON(!fq); + fq_state_change(p, fq, msg, verb); + if (fq->cb.fqs) + fq->cb.fqs(p, fq, msg); + break; + case QM_MR_VERB_FQPN: + /* Parked */ +#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP + fq = get_fq_table_entry(msg->fq.contextB); +#else + fq = (void *)(uintptr_t)msg->fq.contextB; +#endif + fq_state_change(p, fq, msg, verb); + if (fq->cb.fqs) + fq->cb.fqs(p, fq, msg); + break; + case QM_MR_VERB_DC_ERN: + /* DCP ERN */ + if (p->cb_dc_ern) + p->cb_dc_ern(p, msg); + else if (cb_dc_ern) + cb_dc_ern(p, msg); + else + pr_crit_once("Leaking DCP ERNs!\n"); + break; + default: + pr_crit("Invalid MR verb 0x%02x\n", verb); + } + } else { + /* Its a software ERN */ +#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP + fq = get_fq_table_entry(msg->ern.tag); +#else + fq = (void *)(uintptr_t)msg->ern.tag; +#endif + fq->cb.ern(p, fq, msg); + } + num++; + qm_mr_next(&p->p); + goto mr_loop; +mr_done: + qm_mr_cci_consume(&p->p, num); + } + /* + * QM_PIRQ_CSCI has already been cleared, as part of its specific + * processing. If that interrupt source has meanwhile been re-asserted, + * we mustn't clear it here (or in the top-level interrupt handler). + */ + return is & (QM_PIRQ_EQCI | QM_PIRQ_EQRI | QM_PIRQ_MRI); +} + +/* remove some slowish-path stuff from the "fast path" and make sure it isn't + * inlined. */ +static noinline void clear_vdqcr(struct qman_portal *p, struct qman_fq *fq) +{ + p->vdqcr_owned = NULL; + FQLOCK(fq); + fq_clear(fq, QMAN_FQ_STATE_VDQCR); + FQUNLOCK(fq); + wake_up(&affine_queue); +} + +/* Look: no locks, no irq_save()s, no preempt_disable()s! :-) The only states + * that would conflict with other things if they ran at the same time on the + * same cpu are; + * + * (i) setting/clearing vdqcr_owned, and + * (ii) clearing the NE (Not Empty) flag. + * + * Both are safe. Because; + * + * (i) this clearing can only occur after qman_volatile_dequeue() has set the + * vdqcr_owned field (which it does before setting VDQCR), and + * qman_volatile_dequeue() blocks interrupts and preemption while this is + * done so that we can't interfere. + * (ii) the NE flag is only cleared after qman_retire_fq() has set it, and as + * with (i) that API prevents us from interfering until it's safe. + * + * The good thing is that qman_volatile_dequeue() and qman_retire_fq() run far + * less frequently (ie. per-FQ) than __poll_portal_fast() does, so the nett + * advantage comes from this function not having to "lock" anything at all. + * + * Note also that the callbacks are invoked at points which are safe against the + * above potential conflicts, but that this function itself is not re-entrant + * (this is because the function tracks one end of each FIFO in the portal and + * we do *not* want to lock that). So the consequence is that it is safe for + * user callbacks to call into any QMan API *except* qman_poll() (as that's the + * sole API that could be invoking the callback through this function). + */ +static inline unsigned int __poll_portal_fast(struct qman_portal *p, + unsigned int poll_limit) +{ + const struct qm_dqrr_entry *dq; + struct qman_fq *fq; + enum qman_cb_dqrr_result res; + unsigned int limit = 0; + +loop: + qm_dqrr_pvb_update(&p->p); + dq = qm_dqrr_current(&p->p); + if (!dq) + goto done; + if (dq->stat & QM_DQRR_STAT_UNSCHEDULED) { + /* VDQCR: don't trust contextB as the FQ may have been + * configured for h/w consumption and we're draining it + * post-retirement. */ + fq = p->vdqcr_owned; + /* We only set QMAN_FQ_STATE_NE when retiring, so we only need + * to check for clearing it when doing volatile dequeues. It's + * one less thing to check in the critical path (SDQCR). */ + if (dq->stat & QM_DQRR_STAT_FQ_EMPTY) + fq_clear(fq, QMAN_FQ_STATE_NE); + /* this is duplicated from the SDQCR code, but we have stuff to + * do before *and* after this callback, and we don't want + * multiple if()s in the critical path (SDQCR). */ + res = fq->cb.dqrr(p, fq, dq); + if (res == qman_cb_dqrr_stop) + goto done; + /* Check for VDQCR completion */ + if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED) + clear_vdqcr(p, fq); + } else { + /* SDQCR: contextB points to the FQ */ +#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP + fq = get_fq_table_entry(dq->contextB); +#else + fq = (void *)(uintptr_t)dq->contextB; +#endif + /* Now let the callback do its stuff */ + res = fq->cb.dqrr(p, fq, dq); + /* The callback can request that we exit without consuming this + * entry nor advancing; */ + if (res == qman_cb_dqrr_stop) + goto done; + } + /* Interpret 'dq' from a driver perspective. */ + /* Parking isn't possible unless HELDACTIVE was set. NB, + * FORCEELIGIBLE implies HELDACTIVE, so we only need to + * check for HELDACTIVE to cover both. */ + DPA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) || + (res != qman_cb_dqrr_park)); + /* Defer just means "skip it, I'll consume it myself later on" */ + if (res != qman_cb_dqrr_defer) + qm_dqrr_cdc_consume_1ptr(&p->p, dq, (res == qman_cb_dqrr_park)); + /* Move forward */ + qm_dqrr_next(&p->p); + /* Entry processed and consumed, increment our counter. The callback can + * request that we exit after consuming the entry, and we also exit if + * we reach our processing limit, so loop back only if neither of these + * conditions is met. */ + if ((++limit < poll_limit) && (res != qman_cb_dqrr_consume_stop)) + goto loop; +done: + return limit; +} + +u32 qman_irqsource_get(void) +{ + /* "irqsource" and "poll" APIs mustn't redirect when sharing, they + * should shut the user out if they are not the primary CPU hosting the + * portal. That's why we use the "raw" interface. */ + struct qman_portal *p = get_raw_affine_portal(); + u32 ret = p->irq_sources & QM_PIRQ_VISIBLE; + + put_affine_portal(); + return ret; +} +EXPORT_SYMBOL(qman_irqsource_get); + +int qman_p_irqsource_add(struct qman_portal *p, u32 bits __maybe_unused) +{ + __maybe_unused unsigned long irqflags; + +#ifdef FSL_DPA_PORTAL_SHARE +#ifndef __rtems__ + if (p->sharing_redirect) + return -EINVAL; +#endif /* __rtems__ */ +#endif + PORTAL_IRQ_LOCK(p, irqflags); + set_bits(bits & QM_PIRQ_VISIBLE, &p->irq_sources); + qm_isr_enable_write(&p->p, p->irq_sources); + PORTAL_IRQ_UNLOCK(p, irqflags); + return 0; +} +EXPORT_SYMBOL(qman_p_irqsource_add); + +int qman_irqsource_add(u32 bits __maybe_unused) +{ + struct qman_portal *p = get_raw_affine_portal(); + int ret; + + ret = qman_p_irqsource_add(p, bits); + put_affine_portal(); + return ret; +} +EXPORT_SYMBOL(qman_irqsource_add); + +int qman_p_irqsource_remove(struct qman_portal *p, u32 bits) +{ + __maybe_unused unsigned long irqflags; + u32 ier; + +#ifdef FSL_DPA_PORTAL_SHARE +#ifndef __rtems__ + if (p->sharing_redirect) { + put_affine_portal(); + return -EINVAL; + } +#endif /* __rtems__ */ +#endif + /* Our interrupt handler only processes+clears status register bits that + * are in p->irq_sources. As we're trimming that mask, if one of them + * were to assert in the status register just before we remove it from + * the enable register, there would be an interrupt-storm when we + * release the IRQ lock. So we wait for the enable register update to + * take effect in h/w (by reading it back) and then clear all other bits + * in the status register. Ie. we clear them from ISR once it's certain + * IER won't allow them to reassert. */ + PORTAL_IRQ_LOCK(p, irqflags); + bits &= QM_PIRQ_VISIBLE; + clear_bits(bits, &p->irq_sources); + qm_isr_enable_write(&p->p, p->irq_sources); + ier = qm_isr_enable_read(&p->p); + /* Using "~ier" (rather than "bits" or "~p->irq_sources") creates a + * data-dependency, ie. to protect against re-ordering. */ + qm_isr_status_clear(&p->p, ~ier); + PORTAL_IRQ_UNLOCK(p, irqflags); + return 0; +} +EXPORT_SYMBOL(qman_p_irqsource_remove); + +int qman_irqsource_remove(u32 bits) +{ + struct qman_portal *p = get_raw_affine_portal(); + int ret; + + ret = qman_p_irqsource_remove(p, bits); + put_affine_portal(); + return ret; +} +EXPORT_SYMBOL(qman_irqsource_remove); + +#ifndef __rtems__ +const cpumask_t *qman_affine_cpus(void) +{ + return &affine_mask; +} +EXPORT_SYMBOL(qman_affine_cpus); + +u16 qman_affine_channel(int cpu) +{ + if (cpu < 0) { + struct qman_portal *portal = get_raw_affine_portal(); + +#ifdef FSL_DPA_PORTAL_SHARE +#ifndef __rtems__ + BUG_ON(portal->sharing_redirect); +#endif /* __rtems__ */ +#endif + cpu = portal->config->public_cfg.cpu; + put_affine_portal(); + } + BUG_ON(!cpumask_test_cpu(cpu, &affine_mask)); + return affine_channels[cpu]; +} +EXPORT_SYMBOL(qman_affine_channel); +#endif /* __rtems__ */ + +void *qman_get_affine_portal(int cpu) +{ +#ifndef __rtems__ + return affine_portals[cpu]; +#else /* __rtems__ */ + return &per_cpu(qman_affine_portal, cpu); +#endif /* __rtems__ */ +} +EXPORT_SYMBOL(qman_get_affine_portal); + +int qman_p_poll_dqrr(struct qman_portal *p, unsigned int limit) +{ + int ret; + +#ifdef FSL_DPA_PORTAL_SHARE +#ifndef __rtems__ + if (unlikely(p->sharing_redirect)) + ret = -EINVAL; + else +#endif /* __rtems__ */ +#endif + { + BUG_ON(p->irq_sources & QM_PIRQ_DQRI); + ret = __poll_portal_fast(p, limit); + } + return ret; +} +EXPORT_SYMBOL(qman_p_poll_dqrr); + +int qman_poll_dqrr(unsigned int limit) +{ + struct qman_portal *p = get_poll_portal(); + int ret; + + ret = qman_p_poll_dqrr(p, limit); + put_poll_portal(); + return ret; +} +EXPORT_SYMBOL(qman_poll_dqrr); + +u32 qman_p_poll_slow(struct qman_portal *p) +{ + u32 ret; + +#ifdef FSL_DPA_PORTAL_SHARE +#ifndef __rtems__ + if (unlikely(p->sharing_redirect)) + ret = (u32)-1; + else +#endif /* __rtems__ */ +#endif + { + u32 is = qm_isr_status_read(&p->p) & ~p->irq_sources; + + ret = __poll_portal_slow(p, is); + qm_isr_status_clear(&p->p, ret); + } + return ret; +} +EXPORT_SYMBOL(qman_p_poll_slow); + +u32 qman_poll_slow(void) +{ + struct qman_portal *p = get_poll_portal(); + u32 ret; + + ret = qman_p_poll_slow(p); + put_poll_portal(); + return ret; +} +EXPORT_SYMBOL(qman_poll_slow); + +/* Legacy wrapper */ +void qman_p_poll(struct qman_portal *p) +{ +#ifdef FSL_DPA_PORTAL_SHARE +#ifndef __rtems__ + if (unlikely(p->sharing_redirect)) + return; +#endif /* __rtems__ */ +#endif + if ((~p->irq_sources) & QM_PIRQ_SLOW) { + if (!(p->slowpoll--)) { + u32 is = qm_isr_status_read(&p->p) & ~p->irq_sources; + u32 active = __poll_portal_slow(p, is); + + if (active) { + qm_isr_status_clear(&p->p, active); + p->slowpoll = SLOW_POLL_BUSY; + } else + p->slowpoll = SLOW_POLL_IDLE; + } + } + if ((~p->irq_sources) & QM_PIRQ_DQRI) + __poll_portal_fast(p, QMAN_POLL_LIMIT); +} +EXPORT_SYMBOL(qman_p_poll); + +void qman_poll(void) +{ + struct qman_portal *p = get_poll_portal(); + + qman_p_poll(p); + put_poll_portal(); +} +EXPORT_SYMBOL(qman_poll); + +void qman_p_stop_dequeues(struct qman_portal *p) +{ + qman_stop_dequeues_ex(p); +} +EXPORT_SYMBOL(qman_p_stop_dequeues); + +void qman_stop_dequeues(void) +{ + struct qman_portal *p = get_affine_portal(); + + qman_p_stop_dequeues(p); + put_affine_portal(); +} +EXPORT_SYMBOL(qman_stop_dequeues); + +void qman_p_start_dequeues(struct qman_portal *p) +{ + unsigned long irqflags __maybe_unused; + + PORTAL_IRQ_LOCK(p, irqflags); + DPA_ASSERT(p->dqrr_disable_ref > 0); + if (!(--p->dqrr_disable_ref)) + qm_dqrr_set_maxfill(&p->p, DQRR_MAXFILL); + PORTAL_IRQ_UNLOCK(p, irqflags); +} +EXPORT_SYMBOL(qman_p_start_dequeues); + +void qman_start_dequeues(void) +{ + struct qman_portal *p = get_affine_portal(); + + qman_p_start_dequeues(p); + put_affine_portal(); +} +EXPORT_SYMBOL(qman_start_dequeues); + +void qman_p_static_dequeue_add(struct qman_portal *p, u32 pools) +{ + unsigned long irqflags __maybe_unused; + + PORTAL_IRQ_LOCK(p, irqflags); + pools &= p->config->public_cfg.pools; + p->sdqcr |= pools; + qm_dqrr_sdqcr_set(&p->p, p->sdqcr); + PORTAL_IRQ_UNLOCK(p, irqflags); +} +EXPORT_SYMBOL(qman_p_static_dequeue_add); + +void qman_static_dequeue_add(u32 pools) +{ + struct qman_portal *p = get_affine_portal(); + + qman_p_static_dequeue_add(p, pools); + put_affine_portal(); +} +EXPORT_SYMBOL(qman_static_dequeue_add); + +void qman_p_static_dequeue_del(struct qman_portal *p, u32 pools) +{ + unsigned long irqflags __maybe_unused; + + PORTAL_IRQ_LOCK(p, irqflags); + pools &= p->config->public_cfg.pools; + p->sdqcr &= ~pools; + qm_dqrr_sdqcr_set(&p->p, p->sdqcr); + PORTAL_IRQ_UNLOCK(p, irqflags); +} +EXPORT_SYMBOL(qman_p_static_dequeue_del); + +void qman_static_dequeue_del(u32 pools) +{ + struct qman_portal *p = get_affine_portal(); + + qman_p_static_dequeue_del(p, pools); + put_affine_portal(); +} +EXPORT_SYMBOL(qman_static_dequeue_del); + +u32 qman_p_static_dequeue_get(struct qman_portal *p) +{ + return p->sdqcr; +} +EXPORT_SYMBOL(qman_p_static_dequeue_get); + +u32 qman_static_dequeue_get(void) +{ + struct qman_portal *p = get_affine_portal(); + u32 ret = qman_p_static_dequeue_get(p); + + put_affine_portal(); + return ret; +} +EXPORT_SYMBOL(qman_static_dequeue_get); + +void qman_p_dca(struct qman_portal *p, struct qm_dqrr_entry *dq, + int park_request) +{ + qm_dqrr_cdc_consume_1ptr(&p->p, dq, park_request); +} +EXPORT_SYMBOL(qman_p_dca); + +void qman_dca(struct qm_dqrr_entry *dq, int park_request) +{ + struct qman_portal *p = get_affine_portal(); + + qman_p_dca(p, dq, park_request); + put_affine_portal(); +} +EXPORT_SYMBOL(qman_dca); + +/* Frame queue API */ + +static const char *mcr_result_str(u8 result) +{ + switch (result) { + case QM_MCR_RESULT_NULL: + return "QM_MCR_RESULT_NULL"; + case QM_MCR_RESULT_OK: + return "QM_MCR_RESULT_OK"; + case QM_MCR_RESULT_ERR_FQID: + return "QM_MCR_RESULT_ERR_FQID"; + case QM_MCR_RESULT_ERR_FQSTATE: + return "QM_MCR_RESULT_ERR_FQSTATE"; + case QM_MCR_RESULT_ERR_NOTEMPTY: + return "QM_MCR_RESULT_ERR_NOTEMPTY"; + case QM_MCR_RESULT_PENDING: + return "QM_MCR_RESULT_PENDING"; + case QM_MCR_RESULT_ERR_BADCOMMAND: + return "QM_MCR_RESULT_ERR_BADCOMMAND"; + } + return "<unknown MCR result>"; +} + +int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq) +{ + struct qm_fqd fqd; + struct qm_mcr_queryfq_np np; + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p; + unsigned long irqflags __maybe_unused; + + if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) { + int ret = qman_alloc_fqid(&fqid); + + if (ret) + return ret; + } + spin_lock_init(&fq->fqlock); + fq->fqid = fqid; + fq->flags = flags; + fq->state = qman_fq_state_oos; + fq->cgr_groupid = 0; +#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP + if (unlikely(find_empty_fq_table_entry(&fq->key, fq))) + return -ENOMEM; +#endif + if (!(flags & QMAN_FQ_FLAG_AS_IS) || (flags & QMAN_FQ_FLAG_NO_MODIFY)) + return 0; + /* Everything else is AS_IS support */ + p = get_affine_portal(); + PORTAL_IRQ_LOCK(p, irqflags); + mcc = qm_mc_start(&p->p); + mcc->queryfq.fqid = fqid; + qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ); + if (mcr->result != QM_MCR_RESULT_OK) { + pr_err("QUERYFQ failed: %s\n", mcr_result_str(mcr->result)); + goto err; + } + fqd = mcr->queryfq.fqd; + mcc = qm_mc_start(&p->p); + mcc->queryfq_np.fqid = fqid; + qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ_NP); + if (mcr->result != QM_MCR_RESULT_OK) { + pr_err("QUERYFQ_NP failed: %s\n", mcr_result_str(mcr->result)); + goto err; + } + np = mcr->queryfq_np; + /* Phew, have queryfq and queryfq_np results, stitch together + * the FQ object from those. */ + fq->cgr_groupid = fqd.cgid; + switch (np.state & QM_MCR_NP_STATE_MASK) { + case QM_MCR_NP_STATE_OOS: + break; + case QM_MCR_NP_STATE_RETIRED: + fq->state = qman_fq_state_retired; + if (np.frm_cnt) + fq_set(fq, QMAN_FQ_STATE_NE); + break; + case QM_MCR_NP_STATE_TEN_SCHED: + case QM_MCR_NP_STATE_TRU_SCHED: + case QM_MCR_NP_STATE_ACTIVE: + fq->state = qman_fq_state_sched; + if (np.state & QM_MCR_NP_STATE_R) + fq_set(fq, QMAN_FQ_STATE_CHANGING); + break; + case QM_MCR_NP_STATE_PARKED: + fq->state = qman_fq_state_parked; + break; + default: + DPA_ASSERT(NULL == "invalid FQ state"); + } + if (fqd.fq_ctrl & QM_FQCTRL_CGE) + fq->state |= QMAN_FQ_STATE_CGR_EN; + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + return 0; +err: + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) + qman_release_fqid(fqid); + return -EIO; +} +EXPORT_SYMBOL(qman_create_fq); + +void qman_destroy_fq(struct qman_fq *fq, u32 flags __maybe_unused) +{ + + /* We don't need to lock the FQ as it is a pre-condition that the FQ be + * quiesced. Instead, run some checks. */ + switch (fq->state) { + case qman_fq_state_parked: + DPA_ASSERT(flags & QMAN_FQ_DESTROY_PARKED); + case qman_fq_state_oos: + if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID)) + qman_release_fqid(fq->fqid); +#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP + clear_fq_table_entry(fq->key); +#endif + return; + default: + break; + } + DPA_ASSERT(NULL == "qman_free_fq() on unquiesced FQ!"); +} +EXPORT_SYMBOL(qman_destroy_fq); + +u32 qman_fq_fqid(struct qman_fq *fq) +{ + return fq->fqid; +} +EXPORT_SYMBOL(qman_fq_fqid); + +void qman_fq_state(struct qman_fq *fq, enum qman_fq_state *state, u32 *flags) +{ + if (state) + *state = fq->state; + if (flags) + *flags = fq->flags; +} +EXPORT_SYMBOL(qman_fq_state); + +int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts) +{ + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p; + unsigned long irqflags __maybe_unused; + u8 res, myverb = (flags & QMAN_INITFQ_FLAG_SCHED) ? + QM_MCC_VERB_INITFQ_SCHED : QM_MCC_VERB_INITFQ_PARKED; + + if ((fq->state != qman_fq_state_oos) && + (fq->state != qman_fq_state_parked)) + return -EINVAL; +#ifdef CONFIG_FSL_DPA_CHECKING + if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) + return -EINVAL; +#endif + if (opts && (opts->we_mask & QM_INITFQ_WE_OAC)) { + /* And can't be set at the same time as TDTHRESH */ + if (opts->we_mask & QM_INITFQ_WE_TDTHRESH) + return -EINVAL; + } + /* Issue an INITFQ_[PARKED|SCHED] management command */ + p = get_affine_portal(); + PORTAL_IRQ_LOCK(p, irqflags); + FQLOCK(fq); + if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || + ((fq->state != qman_fq_state_oos) && + (fq->state != qman_fq_state_parked)))) { + FQUNLOCK(fq); + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + return -EBUSY; + } + mcc = qm_mc_start(&p->p); + if (opts) + mcc->initfq = *opts; + mcc->initfq.fqid = fq->fqid; + mcc->initfq.count = 0; + /* If the FQ does *not* have the TO_DCPORTAL flag, contextB is set as a + * demux pointer. Otherwise, the caller-provided value is allowed to + * stand, don't overwrite it. */ + if (fq_isclear(fq, QMAN_FQ_FLAG_TO_DCPORTAL)) { + dma_addr_t phys_fq; + + mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTB; +#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP + mcc->initfq.fqd.context_b = fq->key; +#else + mcc->initfq.fqd.context_b = (u32)(uintptr_t)fq; +#endif + /* and the physical address - NB, if the user wasn't trying to + * set CONTEXTA, clear the stashing settings. */ + if (!(mcc->initfq.we_mask & QM_INITFQ_WE_CONTEXTA)) { + mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTA; + memset(&mcc->initfq.fqd.context_a, 0, + sizeof(mcc->initfq.fqd.context_a)); + } else { +#ifndef __rtems__ + phys_fq = dma_map_single(&p->pdev->dev, fq, sizeof(*fq), + DMA_TO_DEVICE); +#else /* __rtems__ */ + phys_fq = (dma_addr_t)fq; +#endif /* __rtems__ */ + qm_fqd_stashing_set64(&mcc->initfq.fqd, phys_fq); + } + } + if (flags & QMAN_INITFQ_FLAG_LOCAL) { + mcc->initfq.fqd.dest.channel = p->config->public_cfg.channel; + if (!(mcc->initfq.we_mask & QM_INITFQ_WE_DESTWQ)) { + mcc->initfq.we_mask |= QM_INITFQ_WE_DESTWQ; + mcc->initfq.fqd.dest.wq = 4; + } + } + qm_mc_commit(&p->p, myverb); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); + res = mcr->result; + if (res != QM_MCR_RESULT_OK) { + FQUNLOCK(fq); + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + return -EIO; + } + if (opts) { + if (opts->we_mask & QM_INITFQ_WE_FQCTRL) { + if (opts->fqd.fq_ctrl & QM_FQCTRL_CGE) + fq_set(fq, QMAN_FQ_STATE_CGR_EN); + else + fq_clear(fq, QMAN_FQ_STATE_CGR_EN); + } + if (opts->we_mask & QM_INITFQ_WE_CGID) + fq->cgr_groupid = opts->fqd.cgid; + } + fq->state = (flags & QMAN_INITFQ_FLAG_SCHED) ? + qman_fq_state_sched : qman_fq_state_parked; + FQUNLOCK(fq); + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + return 0; +} +EXPORT_SYMBOL(qman_init_fq); + +int qman_schedule_fq(struct qman_fq *fq) +{ + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p; + unsigned long irqflags __maybe_unused; + int ret = 0; + u8 res; + + if (fq->state != qman_fq_state_parked) + return -EINVAL; +#ifdef CONFIG_FSL_DPA_CHECKING + if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) + return -EINVAL; +#endif + /* Issue a ALTERFQ_SCHED management command */ + p = get_affine_portal(); + PORTAL_IRQ_LOCK(p, irqflags); + FQLOCK(fq); + if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || + (fq->state != qman_fq_state_parked))) { + ret = -EBUSY; + goto out; + } + mcc = qm_mc_start(&p->p); + mcc->alterfq.fqid = fq->fqid; + qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_SCHED); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_SCHED); + res = mcr->result; + if (res != QM_MCR_RESULT_OK) { + ret = -EIO; + goto out; + } + fq->state = qman_fq_state_sched; +out: + FQUNLOCK(fq); + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + return ret; +} +EXPORT_SYMBOL(qman_schedule_fq); + +int qman_retire_fq(struct qman_fq *fq, u32 *flags) +{ + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p; + unsigned long irqflags __maybe_unused; + int rval; + u8 res; + + if ((fq->state != qman_fq_state_parked) && + (fq->state != qman_fq_state_sched)) + return -EINVAL; +#ifdef CONFIG_FSL_DPA_CHECKING + if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) + return -EINVAL; +#endif + p = get_affine_portal(); + PORTAL_IRQ_LOCK(p, irqflags); + FQLOCK(fq); + if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || + (fq->state == qman_fq_state_retired) || + (fq->state == qman_fq_state_oos))) { + rval = -EBUSY; + goto out; + } + rval = table_push_fq(p, fq); + if (rval) + goto out; + mcc = qm_mc_start(&p->p); + mcc->alterfq.fqid = fq->fqid; + qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_RETIRE); + res = mcr->result; + /* "Elegant" would be to treat OK/PENDING the same way; set CHANGING, + * and defer the flags until FQRNI or FQRN (respectively) show up. But + * "Friendly" is to process OK immediately, and not set CHANGING. We do + * friendly, otherwise the caller doesn't necessarily have a fully + * "retired" FQ on return even if the retirement was immediate. However + * this does mean some code duplication between here and + * fq_state_change(). */ + if (likely(res == QM_MCR_RESULT_OK)) { + rval = 0; + /* Process 'fq' right away, we'll ignore FQRNI */ + if (mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) + fq_set(fq, QMAN_FQ_STATE_NE); + if (mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT) + fq_set(fq, QMAN_FQ_STATE_ORL); + else + table_del_fq(p, fq); + if (flags) + *flags = fq->flags; + fq->state = qman_fq_state_retired; + if (fq->cb.fqs) { + /* Another issue with supporting "immediate" retirement + * is that we're forced to drop FQRNIs, because by the + * time they're seen it may already be "too late" (the + * fq may have been OOS'd and free()'d already). But if + * the upper layer wants a callback whether it's + * immediate or not, we have to fake a "MR" entry to + * look like an FQRNI... */ + struct qm_mr_entry msg; + + msg.verb = QM_MR_VERB_FQRNI; + msg.fq.fqs = mcr->alterfq.fqs; + msg.fq.fqid = fq->fqid; +#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP + msg.fq.contextB = fq->key; +#else + msg.fq.contextB = (u32)(uintptr_t)fq; +#endif + fq->cb.fqs(p, fq, &msg); + } + } else if (res == QM_MCR_RESULT_PENDING) { + rval = 1; + fq_set(fq, QMAN_FQ_STATE_CHANGING); + } else { + rval = -EIO; + table_del_fq(p, fq); + } +out: + FQUNLOCK(fq); + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + return rval; +} +EXPORT_SYMBOL(qman_retire_fq); + +int qman_oos_fq(struct qman_fq *fq) +{ + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p; + unsigned long irqflags __maybe_unused; + int ret = 0; + u8 res; + + if (fq->state != qman_fq_state_retired) + return -EINVAL; +#ifdef CONFIG_FSL_DPA_CHECKING + if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) + return -EINVAL; +#endif + p = get_affine_portal(); + PORTAL_IRQ_LOCK(p, irqflags); + FQLOCK(fq); + if (unlikely((fq_isset(fq, QMAN_FQ_STATE_BLOCKOOS)) || + (fq->state != qman_fq_state_retired))) { + ret = -EBUSY; + goto out; + } + mcc = qm_mc_start(&p->p); + mcc->alterfq.fqid = fq->fqid; + qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_OOS); + res = mcr->result; + if (res != QM_MCR_RESULT_OK) { + ret = -EIO; + goto out; + } + fq->state = qman_fq_state_oos; +out: + FQUNLOCK(fq); + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + return ret; +} +EXPORT_SYMBOL(qman_oos_fq); + +int qman_fq_flow_control(struct qman_fq *fq, int xon) +{ + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p; + unsigned long irqflags __maybe_unused; + int ret = 0; + u8 res; + u8 myverb; + + if ((fq->state == qman_fq_state_oos) || + (fq->state == qman_fq_state_retired) || + (fq->state == qman_fq_state_parked)) + return -EINVAL; + +#ifdef CONFIG_FSL_DPA_CHECKING + if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) + return -EINVAL; +#endif + /* Issue a ALTER_FQXON or ALTER_FQXOFF management command */ + p = get_affine_portal(); + PORTAL_IRQ_LOCK(p, irqflags); + FQLOCK(fq); + if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || + (fq->state == qman_fq_state_parked) || + (fq->state == qman_fq_state_oos) || + (fq->state == qman_fq_state_retired))) { + ret = -EBUSY; + goto out; + } + mcc = qm_mc_start(&p->p); + mcc->alterfq.fqid = fq->fqid; + mcc->alterfq.count = 0; + myverb = xon ? QM_MCC_VERB_ALTER_FQXON : QM_MCC_VERB_ALTER_FQXOFF; + + qm_mc_commit(&p->p, myverb); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); + + res = mcr->result; + if (res != QM_MCR_RESULT_OK) { + ret = -EIO; + goto out; + } +out: + FQUNLOCK(fq); + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + return ret; +} +EXPORT_SYMBOL(qman_fq_flow_control); + +int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd) +{ + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p = get_affine_portal(); + unsigned long irqflags __maybe_unused; + u8 res; + + PORTAL_IRQ_LOCK(p, irqflags); + mcc = qm_mc_start(&p->p); + mcc->queryfq.fqid = fq->fqid; + qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ); + res = mcr->result; + if (res == QM_MCR_RESULT_OK) + *fqd = mcr->queryfq.fqd; + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + if (res != QM_MCR_RESULT_OK) + return -EIO; + return 0; +} +EXPORT_SYMBOL(qman_query_fq); + +int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np) +{ + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p = get_affine_portal(); + unsigned long irqflags __maybe_unused; + u8 res; + + PORTAL_IRQ_LOCK(p, irqflags); + mcc = qm_mc_start(&p->p); + mcc->queryfq.fqid = fq->fqid; + qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP); + res = mcr->result; + if (res == QM_MCR_RESULT_OK) + *np = mcr->queryfq_np; + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + if (res == QM_MCR_RESULT_ERR_FQID) + return -ERANGE; + else if (res != QM_MCR_RESULT_OK) + return -EIO; + return 0; +} +EXPORT_SYMBOL(qman_query_fq_np); + +int qman_query_wq(u8 query_dedicated, struct qm_mcr_querywq *wq) +{ + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p = get_affine_portal(); + unsigned long irqflags __maybe_unused; + u8 res, myverb; + + PORTAL_IRQ_LOCK(p, irqflags); + myverb = (query_dedicated) ? QM_MCR_VERB_QUERYWQ_DEDICATED : + QM_MCR_VERB_QUERYWQ; + mcc = qm_mc_start(&p->p); + mcc->querywq.channel.id = wq->channel.id; + qm_mc_commit(&p->p, myverb); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); + res = mcr->result; + if (res == QM_MCR_RESULT_OK) + *wq = mcr->querywq; + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + if (res != QM_MCR_RESULT_OK) { + pr_err("QUERYWQ failed: %s\n", mcr_result_str(res)); + return -EIO; + } + return 0; +} +EXPORT_SYMBOL(qman_query_wq); + +int qman_testwrite_cgr(struct qman_cgr *cgr, u64 i_bcnt, + struct qm_mcr_cgrtestwrite *result) +{ + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p = get_affine_portal(); + unsigned long irqflags __maybe_unused; + u8 res; + + PORTAL_IRQ_LOCK(p, irqflags); + mcc = qm_mc_start(&p->p); + mcc->cgrtestwrite.cgid = cgr->cgrid; + mcc->cgrtestwrite.i_bcnt_hi = (u8)(i_bcnt >> 32); + mcc->cgrtestwrite.i_bcnt_lo = (u32)i_bcnt; + qm_mc_commit(&p->p, QM_MCC_VERB_CGRTESTWRITE); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_CGRTESTWRITE); + res = mcr->result; + if (res == QM_MCR_RESULT_OK) + *result = mcr->cgrtestwrite; + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + if (res != QM_MCR_RESULT_OK) { + pr_err("CGR TEST WRITE failed: %s\n", mcr_result_str(res)); + return -EIO; + } + return 0; +} +EXPORT_SYMBOL(qman_testwrite_cgr); + +int qman_query_cgr(struct qman_cgr *cgr, struct qm_mcr_querycgr *cgrd) +{ + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p = get_affine_portal(); + unsigned long irqflags __maybe_unused; + u8 res; + + PORTAL_IRQ_LOCK(p, irqflags); + mcc = qm_mc_start(&p->p); + mcc->querycgr.cgid = cgr->cgrid; + qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCGR); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYCGR); + res = mcr->result; + if (res == QM_MCR_RESULT_OK) + *cgrd = mcr->querycgr; + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + if (res != QM_MCR_RESULT_OK) { + pr_err("QUERY_CGR failed: %s\n", mcr_result_str(res)); + return -EIO; + } + return 0; +} +EXPORT_SYMBOL(qman_query_cgr); + +int qman_query_congestion(struct qm_mcr_querycongestion *congestion) +{ + struct qm_mc_result *mcr; + struct qman_portal *p = get_affine_portal(); + unsigned long irqflags __maybe_unused; + u8 res; + + PORTAL_IRQ_LOCK(p, irqflags); + qm_mc_start(&p->p); + qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == + QM_MCC_VERB_QUERYCONGESTION); + res = mcr->result; + if (res == QM_MCR_RESULT_OK) + *congestion = mcr->querycongestion; + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + if (res != QM_MCR_RESULT_OK) { + pr_err("QUERY_CONGESTION failed: %s\n", mcr_result_str(res)); + return -EIO; + } + return 0; +} +EXPORT_SYMBOL(qman_query_congestion); + +/* internal function used as a wait_event() expression */ +static int set_p_vdqcr(struct qman_portal *p, struct qman_fq *fq, u32 vdqcr) +{ + unsigned long irqflags __maybe_unused; + int ret = -EBUSY; + + PORTAL_IRQ_LOCK(p, irqflags); + if (!p->vdqcr_owned) { + FQLOCK(fq); + if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) + goto escape; + fq_set(fq, QMAN_FQ_STATE_VDQCR); + FQUNLOCK(fq); + p->vdqcr_owned = fq; + ret = 0; + } +escape: + PORTAL_IRQ_UNLOCK(p, irqflags); + if (!ret) + qm_dqrr_vdqcr_set(&p->p, vdqcr); + return ret; +} + +static int set_vdqcr(struct qman_portal **p, struct qman_fq *fq, u32 vdqcr) +{ + int ret; + + *p = get_affine_portal(); + ret = set_p_vdqcr(*p, fq, vdqcr); + put_affine_portal(); + return ret; +} + +#ifdef FSL_DPA_CAN_WAIT +static int wait_p_vdqcr_start(struct qman_portal *p, struct qman_fq *fq, + u32 vdqcr, u32 flags) +{ + int ret = 0; + +#ifndef __rtems__ + if (flags & QMAN_VOLATILE_FLAG_WAIT_INT) + ret = wait_event_interruptible(affine_queue, + !(ret = set_p_vdqcr(p, fq, vdqcr))); + else +#endif /* __rtems__ */ + wait_event(affine_queue, !(ret = set_p_vdqcr(p, fq, vdqcr))); + return ret; +} + +static int wait_vdqcr_start(struct qman_portal **p, struct qman_fq *fq, + u32 vdqcr, u32 flags) +{ + int ret = 0; + +#ifndef __rtems__ + if (flags & QMAN_VOLATILE_FLAG_WAIT_INT) + ret = wait_event_interruptible(affine_queue, + !(ret = set_vdqcr(p, fq, vdqcr))); + else +#endif /* __rtems__ */ + wait_event(affine_queue, !(ret = set_vdqcr(p, fq, vdqcr))); + return ret; +} +#endif + +int qman_p_volatile_dequeue(struct qman_portal *p, struct qman_fq *fq, + u32 flags __maybe_unused, u32 vdqcr) +{ + int ret; + + if ((fq->state != qman_fq_state_parked) && + (fq->state != qman_fq_state_retired)) + return -EINVAL; + if (vdqcr & QM_VDQCR_FQID_MASK) + return -EINVAL; + if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) + return -EBUSY; + vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid; +#ifdef FSL_DPA_CAN_WAIT + if (flags & QMAN_VOLATILE_FLAG_WAIT) + ret = wait_p_vdqcr_start(p, fq, vdqcr, flags); + else +#endif + ret = set_p_vdqcr(p, fq, vdqcr); + if (ret) + return ret; + /* VDQCR is set */ +#ifdef FSL_DPA_CAN_WAIT + if (flags & QMAN_VOLATILE_FLAG_FINISH) { +#ifndef __rtems__ + if (flags & QMAN_VOLATILE_FLAG_WAIT_INT) + /* NB: don't propagate any error - the caller wouldn't + * know whether the VDQCR was issued or not. A signal + * could arrive after returning anyway, so the caller + * can check signal_pending() if that's an issue. */ + wait_event_interruptible(affine_queue, + !fq_isset(fq, QMAN_FQ_STATE_VDQCR)); + else +#endif /* __rtems__ */ + wait_event(affine_queue, + !fq_isset(fq, QMAN_FQ_STATE_VDQCR)); + } +#endif + return 0; +} +EXPORT_SYMBOL(qman_p_volatile_dequeue); + +int qman_volatile_dequeue(struct qman_fq *fq, u32 flags __maybe_unused, + u32 vdqcr) +{ + struct qman_portal *p; + int ret; + + if ((fq->state != qman_fq_state_parked) && + (fq->state != qman_fq_state_retired)) + return -EINVAL; + if (vdqcr & QM_VDQCR_FQID_MASK) + return -EINVAL; + if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) + return -EBUSY; + vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid; +#ifdef FSL_DPA_CAN_WAIT + if (flags & QMAN_VOLATILE_FLAG_WAIT) + ret = wait_vdqcr_start(&p, fq, vdqcr, flags); + else +#endif + ret = set_vdqcr(&p, fq, vdqcr); + if (ret) + return ret; + /* VDQCR is set */ +#ifdef FSL_DPA_CAN_WAIT + if (flags & QMAN_VOLATILE_FLAG_FINISH) { +#ifndef __rtems__ + if (flags & QMAN_VOLATILE_FLAG_WAIT_INT) + /* NB: don't propagate any error - the caller wouldn't + * know whether the VDQCR was issued or not. A signal + * could arrive after returning anyway, so the caller + * can check signal_pending() if that's an issue. */ + wait_event_interruptible(affine_queue, + !fq_isset(fq, QMAN_FQ_STATE_VDQCR)); + else +#endif /* __rtems__ */ + wait_event(affine_queue, + !fq_isset(fq, QMAN_FQ_STATE_VDQCR)); + } +#endif + return 0; +} +EXPORT_SYMBOL(qman_volatile_dequeue); + +static noinline void update_eqcr_ci(struct qman_portal *p, u8 avail) +{ + if (avail) + qm_eqcr_cce_prefetch(&p->p); + else + qm_eqcr_cce_update(&p->p); +} + +int qman_eqcr_is_empty(void) +{ + unsigned long irqflags __maybe_unused; + struct qman_portal *p = get_affine_portal(); + u8 avail; + + PORTAL_IRQ_LOCK(p, irqflags); + update_eqcr_ci(p, 0); + avail = qm_eqcr_get_fill(&p->p); + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + return avail == 0; +} +EXPORT_SYMBOL(qman_eqcr_is_empty); + +void qman_set_dc_ern(qman_cb_dc_ern handler, int affine) +{ + if (affine) { + unsigned long irqflags __maybe_unused; + struct qman_portal *p = get_affine_portal(); + + PORTAL_IRQ_LOCK(p, irqflags); + p->cb_dc_ern = handler; + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + } else + cb_dc_ern = handler; +} +EXPORT_SYMBOL(qman_set_dc_ern); + +static inline struct qm_eqcr_entry *try_p_eq_start(struct qman_portal *p, + unsigned long *irqflags __maybe_unused, + struct qman_fq *fq, + const struct qm_fd *fd, + u32 flags) +{ + struct qm_eqcr_entry *eq; + u8 avail; + + PORTAL_IRQ_LOCK(p, (*irqflags)); +#ifdef FSL_DPA_CAN_WAIT_SYNC + if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) && + (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) { + if (p->eqci_owned) { + PORTAL_IRQ_UNLOCK(p, (*irqflags)); + return NULL; + } + p->eqci_owned = fq; + } +#endif + if (p->use_eqcr_ci_stashing) { + /* + * The stashing case is easy, only update if we need to in + * order to try and liberate ring entries. + */ + eq = qm_eqcr_start_stash(&p->p); + } else { + /* + * The non-stashing case is harder, need to prefetch ahead of + * time. + */ + avail = qm_eqcr_get_avail(&p->p); + if (avail < 2) + update_eqcr_ci(p, avail); + eq = qm_eqcr_start_no_stash(&p->p); + } + + if (unlikely(!eq)) { +#ifdef FSL_DPA_CAN_WAIT_SYNC + if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) && + (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) + p->eqci_owned = NULL; +#endif + PORTAL_IRQ_UNLOCK(p, (*irqflags)); + return NULL; + } + if (flags & QMAN_ENQUEUE_FLAG_DCA) + eq->dca = QM_EQCR_DCA_ENABLE | + ((flags & QMAN_ENQUEUE_FLAG_DCA_PARK) ? + QM_EQCR_DCA_PARK : 0) | + ((flags >> 8) & QM_EQCR_DCA_IDXMASK); + eq->fqid = fq->fqid; +#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP + eq->tag = fq->key; +#else + eq->tag = (u32)(uintptr_t)fq; +#endif + eq->fd = *fd; + return eq; +} + +static inline struct qm_eqcr_entry *try_eq_start(struct qman_portal **p, + unsigned long *irqflags __maybe_unused, + struct qman_fq *fq, + const struct qm_fd *fd, + u32 flags) +{ + struct qm_eqcr_entry *eq; + + *p = get_affine_portal(); + eq = try_p_eq_start(*p, irqflags, fq, fd, flags); + if (!eq) + put_affine_portal(); + return eq; +} + +#ifdef FSL_DPA_CAN_WAIT +static noinline struct qm_eqcr_entry *__wait_eq_start(struct qman_portal **p, + unsigned long *irqflags __maybe_unused, + struct qman_fq *fq, + const struct qm_fd *fd, + u32 flags) +{ + struct qm_eqcr_entry *eq = try_eq_start(p, irqflags, fq, fd, flags); + + if (!eq) + qm_eqcr_set_ithresh(&(*p)->p, EQCR_ITHRESH); + return eq; +} +static noinline struct qm_eqcr_entry *wait_eq_start(struct qman_portal **p, + unsigned long *irqflags __maybe_unused, + struct qman_fq *fq, + const struct qm_fd *fd, + u32 flags) +{ + struct qm_eqcr_entry *eq; + +#ifndef __rtems__ + if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT) + wait_event_interruptible(affine_queue, + (eq = __wait_eq_start(p, irqflags, fq, fd, flags))); + else +#endif /* __rtems__ */ + wait_event(affine_queue, + (eq = __wait_eq_start(p, irqflags, fq, fd, flags))); + return eq; +} +static noinline struct qm_eqcr_entry *__wait_p_eq_start(struct qman_portal *p, + unsigned long *irqflags __maybe_unused, + struct qman_fq *fq, + const struct qm_fd *fd, + u32 flags) +{ + struct qm_eqcr_entry *eq = try_p_eq_start(p, irqflags, fq, fd, flags); + + if (!eq) + qm_eqcr_set_ithresh(&p->p, EQCR_ITHRESH); + return eq; +} +static noinline struct qm_eqcr_entry *wait_p_eq_start(struct qman_portal *p, + unsigned long *irqflags __maybe_unused, + struct qman_fq *fq, + const struct qm_fd *fd, + u32 flags) +{ + struct qm_eqcr_entry *eq; + +#ifndef __rtems__ + if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT) + wait_event_interruptible(affine_queue, + (eq = __wait_p_eq_start(p, irqflags, fq, fd, flags))); + else +#endif /* __rtems__ */ + wait_event(affine_queue, + (eq = __wait_p_eq_start(p, irqflags, fq, fd, flags))); + return eq; +} +#endif + +int qman_p_enqueue(struct qman_portal *p, struct qman_fq *fq, + const struct qm_fd *fd, u32 flags) +{ + struct qm_eqcr_entry *eq; + unsigned long irqflags __maybe_unused; + +#ifdef FSL_DPA_CAN_WAIT + if (flags & QMAN_ENQUEUE_FLAG_WAIT) + eq = wait_p_eq_start(p, &irqflags, fq, fd, flags); + else +#endif + eq = try_p_eq_start(p, &irqflags, fq, fd, flags); + if (!eq) + return -EBUSY; + /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ + qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE | + (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); + /* Factor the below out, it's used from qman_enqueue_orp() too */ + PORTAL_IRQ_UNLOCK(p, irqflags); +#ifdef FSL_DPA_CAN_WAIT_SYNC + if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) && + (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) { +#ifndef __rtems__ + if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT) + wait_event_interruptible(affine_queue, + (p->eqci_owned != fq)); + else +#endif /* __rtems__ */ + wait_event(affine_queue, (p->eqci_owned != fq)); + } +#endif + return 0; +} +EXPORT_SYMBOL(qman_p_enqueue); + +int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd, u32 flags) +{ + struct qman_portal *p; + struct qm_eqcr_entry *eq; + unsigned long irqflags __maybe_unused; + +#ifdef FSL_DPA_CAN_WAIT + if (flags & QMAN_ENQUEUE_FLAG_WAIT) + eq = wait_eq_start(&p, &irqflags, fq, fd, flags); + else +#endif + eq = try_eq_start(&p, &irqflags, fq, fd, flags); + if (!eq) + return -EBUSY; + /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ + qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE | + (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); + /* Factor the below out, it's used from qman_enqueue_orp() too */ + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); +#ifdef FSL_DPA_CAN_WAIT_SYNC + if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) && + (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) { +#ifndef __rtems__ + if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT) + wait_event_interruptible(affine_queue, + (p->eqci_owned != fq)); + else +#endif /* __rtems__ */ + wait_event(affine_queue, (p->eqci_owned != fq)); + } +#endif + return 0; +} +EXPORT_SYMBOL(qman_enqueue); + +int qman_p_enqueue_orp(struct qman_portal *p, struct qman_fq *fq, + const struct qm_fd *fd, u32 flags, + struct qman_fq *orp, u16 orp_seqnum) +{ + struct qm_eqcr_entry *eq; + unsigned long irqflags __maybe_unused; + +#ifdef FSL_DPA_CAN_WAIT + if (flags & QMAN_ENQUEUE_FLAG_WAIT) + eq = wait_p_eq_start(p, &irqflags, fq, fd, flags); + else +#endif + eq = try_p_eq_start(p, &irqflags, fq, fd, flags); + if (!eq) + return -EBUSY; + /* Process ORP-specifics here */ + if (flags & QMAN_ENQUEUE_FLAG_NLIS) + orp_seqnum |= QM_EQCR_SEQNUM_NLIS; + else { + orp_seqnum &= ~QM_EQCR_SEQNUM_NLIS; + if (flags & QMAN_ENQUEUE_FLAG_NESN) + orp_seqnum |= QM_EQCR_SEQNUM_NESN; + else + /* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */ + orp_seqnum &= ~QM_EQCR_SEQNUM_NESN; + } + eq->seqnum = orp_seqnum; + eq->orp = orp->fqid; + /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ + qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_ORP | + ((flags & (QMAN_ENQUEUE_FLAG_HOLE | QMAN_ENQUEUE_FLAG_NESN)) ? + 0 : QM_EQCR_VERB_CMD_ENQUEUE) | + (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); + PORTAL_IRQ_UNLOCK(p, irqflags); +#ifdef FSL_DPA_CAN_WAIT_SYNC + if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) && + (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) { +#ifndef __rtems__ + if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT) + wait_event_interruptible(affine_queue, + (p->eqci_owned != fq)); + else +#endif /* __rtems__ */ + wait_event(affine_queue, (p->eqci_owned != fq)); + } +#endif + return 0; +} +EXPORT_SYMBOL(qman_p_enqueue_orp); + +int qman_enqueue_orp(struct qman_fq *fq, const struct qm_fd *fd, u32 flags, + struct qman_fq *orp, u16 orp_seqnum) +{ + struct qman_portal *p; + struct qm_eqcr_entry *eq; + unsigned long irqflags __maybe_unused; + +#ifdef FSL_DPA_CAN_WAIT + if (flags & QMAN_ENQUEUE_FLAG_WAIT) + eq = wait_eq_start(&p, &irqflags, fq, fd, flags); + else +#endif + eq = try_eq_start(&p, &irqflags, fq, fd, flags); + if (!eq) + return -EBUSY; + /* Process ORP-specifics here */ + if (flags & QMAN_ENQUEUE_FLAG_NLIS) + orp_seqnum |= QM_EQCR_SEQNUM_NLIS; + else { + orp_seqnum &= ~QM_EQCR_SEQNUM_NLIS; + if (flags & QMAN_ENQUEUE_FLAG_NESN) + orp_seqnum |= QM_EQCR_SEQNUM_NESN; + else + /* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */ + orp_seqnum &= ~QM_EQCR_SEQNUM_NESN; + } + eq->seqnum = orp_seqnum; + eq->orp = orp->fqid; + /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ + qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_ORP | + ((flags & (QMAN_ENQUEUE_FLAG_HOLE | QMAN_ENQUEUE_FLAG_NESN)) ? + 0 : QM_EQCR_VERB_CMD_ENQUEUE) | + (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); +#ifdef FSL_DPA_CAN_WAIT_SYNC + if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) && + (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) { +#ifndef __rtems__ + if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT) + wait_event_interruptible(affine_queue, + (p->eqci_owned != fq)); + else +#endif /* __rtems__ */ + wait_event(affine_queue, (p->eqci_owned != fq)); + } +#endif + return 0; +} +EXPORT_SYMBOL(qman_enqueue_orp); + +int qman_p_enqueue_precommit(struct qman_portal *p, struct qman_fq *fq, + const struct qm_fd *fd, u32 flags, + qman_cb_precommit cb, void *cb_arg) +{ + struct qm_eqcr_entry *eq; + unsigned long irqflags __maybe_unused; + +#ifdef FSL_DPA_CAN_WAIT + if (flags & QMAN_ENQUEUE_FLAG_WAIT) + eq = wait_p_eq_start(p, &irqflags, fq, fd, flags); + else +#endif + eq = try_p_eq_start(p, &irqflags, fq, fd, flags); + if (!eq) + return -EBUSY; + /* invoke user supplied callback function before writing commit verb */ + if (cb(cb_arg)) { + PORTAL_IRQ_UNLOCK(p, irqflags); + return -EINVAL; + } + /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ + qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE | + (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); + /* Factor the below out, it's used from qman_enqueue_orp() too */ + PORTAL_IRQ_UNLOCK(p, irqflags); +#ifdef FSL_DPA_CAN_WAIT_SYNC + if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) && + (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) { +#ifndef __rtems__ + if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT) + wait_event_interruptible(affine_queue, + (p->eqci_owned != fq)); + else +#endif /* __rtems__ */ + wait_event(affine_queue, (p->eqci_owned != fq)); + } +#endif + return 0; +} +EXPORT_SYMBOL(qman_p_enqueue_precommit); + +int qman_enqueue_precommit(struct qman_fq *fq, const struct qm_fd *fd, + u32 flags, qman_cb_precommit cb, void *cb_arg) +{ + struct qman_portal *p; + struct qm_eqcr_entry *eq; + unsigned long irqflags __maybe_unused; + +#ifdef FSL_DPA_CAN_WAIT + if (flags & QMAN_ENQUEUE_FLAG_WAIT) + eq = wait_eq_start(&p, &irqflags, fq, fd, flags); + else +#endif + eq = try_eq_start(&p, &irqflags, fq, fd, flags); + if (!eq) + return -EBUSY; + /* invoke user supplied callback function before writing commit verb */ + if (cb(cb_arg)) { + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + return -EINVAL; + } + /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ + qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE | + (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); + /* Factor the below out, it's used from qman_enqueue_orp() too */ + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); +#ifdef FSL_DPA_CAN_WAIT_SYNC + if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) && + (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) { +#ifndef __rtems__ + if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT) + wait_event_interruptible(affine_queue, + (p->eqci_owned != fq)); + else +#endif /* __rtems__ */ + wait_event(affine_queue, (p->eqci_owned != fq)); + } +#endif + return 0; +} +EXPORT_SYMBOL(qman_enqueue_precommit); + +int qman_modify_cgr(struct qman_cgr *cgr, u32 flags, + struct qm_mcc_initcgr *opts) +{ + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p = get_affine_portal(); + unsigned long irqflags __maybe_unused; + u8 res; + u8 verb = QM_MCC_VERB_MODIFYCGR; + + PORTAL_IRQ_LOCK(p, irqflags); + mcc = qm_mc_start(&p->p); + if (opts) + mcc->initcgr = *opts; + mcc->initcgr.cgid = cgr->cgrid; + if (flags & QMAN_CGR_FLAG_USE_INIT) + verb = QM_MCC_VERB_INITCGR; + qm_mc_commit(&p->p, verb); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == verb); + res = mcr->result; + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + return (res == QM_MCR_RESULT_OK) ? 0 : -EIO; +} +EXPORT_SYMBOL(qman_modify_cgr); + +#define TARG_MASK(n) (0x80000000 >> (n->config->public_cfg.channel - \ + QM_CHANNEL_SWPORTAL0)) +#define PORTAL_IDX(n) (n->config->public_cfg.channel - QM_CHANNEL_SWPORTAL0) + +static u8 qman_cgr_cpus[__CGR_NUM]; + +int qman_create_cgr(struct qman_cgr *cgr, u32 flags, + struct qm_mcc_initcgr *opts) +{ + unsigned long irqflags __maybe_unused; + struct qm_mcr_querycgr cgr_state; + struct qm_mcc_initcgr local_opts; + int ret; + struct qman_portal *p; + + /* We have to check that the provided CGRID is within the limits of the + * data-structures, for obvious reasons. However we'll let h/w take + * care of determining whether it's within the limits of what exists on + * the SoC. */ + if (cgr->cgrid >= __CGR_NUM) + return -EINVAL; + + preempt_disable(); + p = get_affine_portal(); + qman_cgr_cpus[cgr->cgrid] = smp_processor_id(); + preempt_enable(); + + memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); + cgr->chan = p->config->public_cfg.channel; + spin_lock_irqsave(&p->cgr_lock, irqflags); + + /* if no opts specified, just add it to the list */ + if (!opts) + goto add_list; + + ret = qman_query_cgr(cgr, &cgr_state); + if (ret) + goto release_lock; + if (opts) + local_opts = *opts; + if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) + local_opts.cgr.cscn_targ_upd_ctrl = + QM_CGR_TARG_UDP_CTRL_WRITE_BIT | PORTAL_IDX(p); + else + /* Overwrite TARG */ + local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ | + TARG_MASK(p); + local_opts.we_mask |= QM_CGR_WE_CSCN_TARG; + + /* send init if flags indicate so */ + if (opts && (flags & QMAN_CGR_FLAG_USE_INIT)) + ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, &local_opts); + else + ret = qman_modify_cgr(cgr, 0, &local_opts); + if (ret) + goto release_lock; +add_list: + list_add(&cgr->node, &p->cgr_cbs); + + /* Determine if newly added object requires its callback to be called */ + ret = qman_query_cgr(cgr, &cgr_state); + if (ret) { + /* we can't go back, so proceed and return success, but screen + * and wail to the log file */ + pr_crit("CGR HW state partially modified\n"); + ret = 0; + goto release_lock; + } + if (cgr->cb && cgr_state.cgr.cscn_en && qman_cgrs_get(&p->cgrs[1], + cgr->cgrid)) + cgr->cb(p, cgr, 1); +release_lock: + spin_unlock_irqrestore(&p->cgr_lock, irqflags); + put_affine_portal(); + return ret; +} +EXPORT_SYMBOL(qman_create_cgr); + +int qman_create_cgr_to_dcp(struct qman_cgr *cgr, u32 flags, u16 dcp_portal, + struct qm_mcc_initcgr *opts) +{ + unsigned long irqflags __maybe_unused; + struct qm_mcc_initcgr local_opts; + int ret; + + if ((qman_ip_rev & 0xFF00) < QMAN_REV30) { + pr_warn("This version doesn't support to send CSCN to DCP portal\n"); + return -EINVAL; + } + /* We have to check that the provided CGRID is within the limits of the + * data-structures, for obvious reasons. However we'll let h/w take + * care of determining whether it's within the limits of what exists on + * the SoC. + */ + if (cgr->cgrid >= __CGR_NUM) + return -EINVAL; + + memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); + if (opts) + local_opts = *opts; + + local_opts.cgr.cscn_targ_upd_ctrl = QM_CGR_TARG_UDP_CTRL_WRITE_BIT | + QM_CGR_TARG_UDP_CTRL_DCP | dcp_portal; + local_opts.we_mask |= QM_CGR_WE_CSCN_TARG; + + /* send init if flags indicate so */ + if (opts && (flags & QMAN_CGR_FLAG_USE_INIT)) + ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, + &local_opts); + else + ret = qman_modify_cgr(cgr, 0, &local_opts); + + return ret; +} +EXPORT_SYMBOL(qman_create_cgr_to_dcp); + +int qman_delete_cgr(struct qman_cgr *cgr) +{ + unsigned long irqflags __maybe_unused; + struct qm_mcr_querycgr cgr_state; + struct qm_mcc_initcgr local_opts; + int ret = 0; + struct qman_cgr *i; + struct qman_portal *p = get_affine_portal(); + + if (cgr->chan != p->config->public_cfg.channel) { + pr_crit("Attempting to delete cgr from different portal " + "than it was create: create 0x%x, delete 0x%x\n", + cgr->chan, p->config->public_cfg.channel); + ret = -EINVAL; + goto put_portal; + } + memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); + spin_lock_irqsave(&p->cgr_lock, irqflags); + list_del(&cgr->node); + /* + * If there are no other CGR objects for this CGRID in the list, update + * CSCN_TARG accordingly + */ + list_for_each_entry(i, &p->cgr_cbs, node) + if ((i->cgrid == cgr->cgrid) && i->cb) + goto release_lock; + ret = qman_query_cgr(cgr, &cgr_state); + if (ret) { + /* add back to the list */ + list_add(&cgr->node, &p->cgr_cbs); + goto release_lock; + } + /* Overwrite TARG */ + local_opts.we_mask = QM_CGR_WE_CSCN_TARG; + if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) + local_opts.cgr.cscn_targ_upd_ctrl = PORTAL_IDX(p); + else + local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ & + ~(TARG_MASK(p)); + ret = qman_modify_cgr(cgr, 0, &local_opts); + if (ret) + /* add back to the list */ + list_add(&cgr->node, &p->cgr_cbs); +release_lock: + spin_unlock_irqrestore(&p->cgr_lock, irqflags); +put_portal: + put_affine_portal(); + return ret; +} +EXPORT_SYMBOL(qman_delete_cgr); + +#ifndef __rtems__ +struct cgr_comp { + struct qman_cgr *cgr; + struct completion completion; +}; + +static int qman_delete_cgr_thread(void *p) +{ + struct cgr_comp *cgr_comp = (struct cgr_comp *)p; + int res; + + res = qman_delete_cgr((struct qman_cgr *)cgr_comp->cgr); + complete(&cgr_comp->completion); + + return res; +} + +void qman_delete_cgr_safe(struct qman_cgr *cgr) +{ + struct task_struct *thread; + struct cgr_comp cgr_comp; + + preempt_disable(); + if (qman_cgr_cpus[cgr->cgrid] != smp_processor_id()) { + init_completion(&cgr_comp.completion); + cgr_comp.cgr = cgr; + thread = kthread_create(qman_delete_cgr_thread, &cgr_comp, + "cgr_del"); + + if (likely(!IS_ERR(thread))) { + kthread_bind(thread, qman_cgr_cpus[cgr->cgrid]); + wake_up_process(thread); + wait_for_completion(&cgr_comp.completion); + preempt_enable(); + return; + } + } + qman_delete_cgr(cgr); + preempt_enable(); +} +EXPORT_SYMBOL(qman_delete_cgr_safe); +#endif /* __rtems__ */ + +int qman_set_wpm(int wpm_enable) +{ + return qm_set_wpm(wpm_enable); +} +EXPORT_SYMBOL(qman_set_wpm); + +int qman_get_wpm(int *wpm_enable) +{ + return qm_get_wpm(wpm_enable); +} +EXPORT_SYMBOL(qman_get_wpm); + + +/* Cleanup FQs */ +static int qm_shutdown_fq(struct qm_portal **portal, int portal_count, + u32 fqid) +{ + + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + u8 state; + int orl_empty, fq_empty, i, drain = 0; + u32 result; + u32 channel, wq; + + /* Determine the state of the FQID */ + mcc = qm_mc_start(portal[0]); + mcc->queryfq_np.fqid = fqid; + qm_mc_commit(portal[0], QM_MCC_VERB_QUERYFQ_NP); + while (!(mcr = qm_mc_result(portal[0]))) + cpu_relax(); + DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP); + state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK; + if (state == QM_MCR_NP_STATE_OOS) + return 0; /* Already OOS, no need to do anymore checks */ + + /* Query which channel the FQ is using */ + mcc = qm_mc_start(portal[0]); + mcc->queryfq.fqid = fqid; + qm_mc_commit(portal[0], QM_MCC_VERB_QUERYFQ); + while (!(mcr = qm_mc_result(portal[0]))) + cpu_relax(); + DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ); + + /* Need to store these since the MCR gets reused */ + channel = mcr->queryfq.fqd.dest.channel; + wq = mcr->queryfq.fqd.dest.wq; + + switch (state) { + case QM_MCR_NP_STATE_TEN_SCHED: + case QM_MCR_NP_STATE_TRU_SCHED: + case QM_MCR_NP_STATE_ACTIVE: + case QM_MCR_NP_STATE_PARKED: + orl_empty = 0; + mcc = qm_mc_start(portal[0]); + mcc->alterfq.fqid = fqid; + qm_mc_commit(portal[0], QM_MCC_VERB_ALTER_RETIRE); + while (!(mcr = qm_mc_result(portal[0]))) + cpu_relax(); + DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == + QM_MCR_VERB_ALTER_RETIRE); + result = mcr->result; /* Make a copy as we reuse MCR below */ + + if (result == QM_MCR_RESULT_PENDING) { + /* Need to wait for the FQRN in the message ring, which + will only occur once the FQ has been drained. In + order for the FQ to drain the portal needs to be set + to dequeue from the channel the FQ is scheduled on */ + const struct qm_mr_entry *msg; + const struct qm_dqrr_entry *dqrr = NULL; + int found_fqrn = 0; + u16 dequeue_wq = 0; + + /* Flag that we need to drain FQ */ + drain = 1; + + if (channel >= qm_channel_pool1 && + channel < (qm_channel_pool1 + 15)) { + /* Pool channel, enable the bit in the portal */ + dequeue_wq = (channel - + qm_channel_pool1 + 1)<<4 | wq; + } else if (channel < qm_channel_pool1) { + /* Dedicated channel */ + dequeue_wq = wq; + } else { + pr_info("Cannot recover FQ 0x%x, it is " + "scheduled on channel 0x%x", + fqid, channel); + return -EBUSY; + } + /* Set the sdqcr to drain this channel */ + if (channel < qm_channel_pool1) + for (i = 0; i < portal_count; i++) + qm_dqrr_sdqcr_set(portal[i], + QM_SDQCR_TYPE_ACTIVE | + QM_SDQCR_CHANNELS_DEDICATED); + else + for (i = 0; i < portal_count; i++) + qm_dqrr_sdqcr_set( + portal[i], + QM_SDQCR_TYPE_ACTIVE | + QM_SDQCR_CHANNELS_POOL_CONV + (channel)); + while (!found_fqrn) { + /* Keep draining DQRR while checking the MR*/ + for (i = 0; i < portal_count; i++) { + qm_dqrr_pvb_update(portal[i]); + dqrr = qm_dqrr_current(portal[i]); + while (dqrr) { + qm_dqrr_cdc_consume_1ptr( + portal[i], dqrr, 0); + qm_dqrr_pvb_update(portal[i]); + qm_dqrr_next(portal[i]); + dqrr = qm_dqrr_current( + portal[i]); + } + /* Process message ring too */ + qm_mr_pvb_update(portal[i]); + msg = qm_mr_current(portal[i]); + while (msg) { + if ((msg->verb & + QM_MR_VERB_TYPE_MASK) + == QM_MR_VERB_FQRN) + found_fqrn = 1; + qm_mr_next(portal[i]); + qm_mr_cci_consume_to_current( + portal[i]); + qm_mr_pvb_update(portal[i]); + msg = qm_mr_current(portal[i]); + } + cpu_relax(); + } + } + } + if (result != QM_MCR_RESULT_OK && + result != QM_MCR_RESULT_PENDING) { + /* error */ + pr_err("qman_retire_fq failed on FQ 0x%x, result=0x%x\n", + fqid, result); + return -1; + } + if (!(mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)) { + /* ORL had no entries, no need to wait until the + ERNs come in */ + orl_empty = 1; + } + /* Retirement succeeded, check to see if FQ needs + to be drained */ + if (drain || mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) { + /* FQ is Not Empty, drain using volatile DQ commands */ + fq_empty = 0; + do { + const struct qm_dqrr_entry *dqrr = NULL; + u32 vdqcr = fqid | QM_VDQCR_NUMFRAMES_SET(3); + qm_dqrr_vdqcr_set(portal[0], vdqcr); + + /* Wait for a dequeue to occur */ + while (dqrr == NULL) { + qm_dqrr_pvb_update(portal[0]); + dqrr = qm_dqrr_current(portal[0]); + if (!dqrr) + cpu_relax(); + } + /* Process the dequeues, making sure to + empty the ring completely */ + while (dqrr) { + if (dqrr->fqid == fqid && + dqrr->stat & QM_DQRR_STAT_FQ_EMPTY) + fq_empty = 1; + qm_dqrr_cdc_consume_1ptr(portal[0], + dqrr, 0); + qm_dqrr_pvb_update(portal[0]); + qm_dqrr_next(portal[0]); + dqrr = qm_dqrr_current(portal[0]); + } + } while (fq_empty == 0); + } + for (i = 0; i < portal_count; i++) + qm_dqrr_sdqcr_set(portal[i], 0); + + /* Wait for the ORL to have been completely drained */ + while (orl_empty == 0) { + const struct qm_mr_entry *msg; + + qm_mr_pvb_update(portal[0]); + msg = qm_mr_current(portal[0]); + while (msg) { + if ((msg->verb & QM_MR_VERB_TYPE_MASK) == + QM_MR_VERB_FQRL) + orl_empty = 1; + qm_mr_next(portal[0]); + qm_mr_cci_consume_to_current(portal[0]); + qm_mr_pvb_update(portal[0]); + msg = qm_mr_current(portal[0]); + } + cpu_relax(); + } + mcc = qm_mc_start(portal[0]); + mcc->alterfq.fqid = fqid; + qm_mc_commit(portal[0], QM_MCC_VERB_ALTER_OOS); + while (!(mcr = qm_mc_result(portal[0]))) + cpu_relax(); + DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == + QM_MCR_VERB_ALTER_OOS); + if (mcr->result != QM_MCR_RESULT_OK) { + pr_err("OOS after drain Failed on FQID 0x%x, result 0x%x\n", + fqid, mcr->result); + return -1; + } + return 0; + case QM_MCR_NP_STATE_RETIRED: + /* Send OOS Command */ + mcc = qm_mc_start(portal[0]); + mcc->alterfq.fqid = fqid; + qm_mc_commit(portal[0], QM_MCC_VERB_ALTER_OOS); + while (!(mcr = qm_mc_result(portal[0]))) + cpu_relax(); + DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == + QM_MCR_VERB_ALTER_OOS); + if (mcr->result) { + pr_err("OOS Failed on FQID 0x%x\n", fqid); + return -1; + } + return 0; + case QM_MCR_NP_STATE_OOS: + /* Done */ + return 0; + } + return -1; +} + +int qman_shutdown_fq(u32 fqid) +{ + struct qman_portal *p; + unsigned long irqflags __maybe_unused; + int ret; + struct qm_portal *low_p; + + p = get_affine_portal(); + PORTAL_IRQ_LOCK(p, irqflags); + low_p = &p->p; + ret = qm_shutdown_fq(&low_p, 1, fqid); + PORTAL_IRQ_UNLOCK(p, irqflags); + put_affine_portal(); + return ret; +} + +const struct qm_portal_config *qman_get_qm_portal_config( + struct qman_portal *portal) +{ +#ifndef __rtems__ + return portal->sharing_redirect ? NULL : portal->config; +#else /* __rtems__ */ + return portal->config; +#endif /* __rtems__ */ +} |