Linux update to 4.11-rc5

Linux baseline a71c9a1c779f2499fb2afc0553e543f18aff6edf (4.11-rc5).

1 files changed, 2633 insertions, 832 deletions

diff --git a/linux/drivers/soc/fsl/qbman/qman.c b/linux/drivers/soc/fsl/qbman/qman.c
index aa188888..244cf164 100644
--- a/linux/drivers/soc/fsl/qbman/qman.c
+++ b/linux/drivers/soc/fsl/qbman/qman.c
@@ -2,7 +2,7 @@
 
 #include <rtems/bsd/local/opt_dpaa.h>
 
-/* Copyright 2008 - 2015 Freescale Semiconductor, Inc.
+/* Copyright 2008 - 2016 Freescale Semiconductor, Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -33,1074 +33,2875 @@
  */
 
 #include "qman_priv.h"
+#ifdef __rtems__
+#undef dev_crit
+#undef dev_dbg
+#undef dev_err
+#define	dev_crit(dev, fmt, ...) printf(fmt, ##__VA_ARGS__)
+#define dev_dbg dev_crit
+#define dev_err dev_crit
+#endif /* __rtems__ */
+
+#define DQRR_MAXFILL	15
+#define EQCR_ITHRESH	4	/* if EQCR congests, interrupt threshold */
+#define IRQNAME		"QMan portal %d"
+#define MAX_IRQNAME	16	/* big enough for "QMan portal %d" */
+#define QMAN_POLL_LIMIT 32
+#define QMAN_PIRQ_DQRR_ITHRESH 12
+#define QMAN_PIRQ_MR_ITHRESH 4
+#define QMAN_PIRQ_IPERIOD 100
+
+/* Portal register assists */
+
+/* Cache-inhibited register offsets */
+#define QM_REG_EQCR_PI_CINH	0x0000
+#define QM_REG_EQCR_CI_CINH	0x0004
+#define QM_REG_EQCR_ITR		0x0008
+#define QM_REG_DQRR_PI_CINH	0x0040
+#define QM_REG_DQRR_CI_CINH	0x0044
+#define QM_REG_DQRR_ITR		0x0048
+#define QM_REG_DQRR_DCAP	0x0050
+#define QM_REG_DQRR_SDQCR	0x0054
+#define QM_REG_DQRR_VDQCR	0x0058
+#define QM_REG_DQRR_PDQCR	0x005c
+#define QM_REG_MR_PI_CINH	0x0080
+#define QM_REG_MR_CI_CINH	0x0084
+#define QM_REG_MR_ITR		0x0088
+#define QM_REG_CFG		0x0100
+#define QM_REG_ISR		0x0e00
+#define QM_REG_IER		0x0e04
+#define QM_REG_ISDR		0x0e08
+#define QM_REG_IIR		0x0e0c
+#define QM_REG_ITPR		0x0e14
+
+/* Cache-enabled register offsets */
+#define QM_CL_EQCR		0x0000
+#define QM_CL_DQRR		0x1000
+#define QM_CL_MR		0x2000
+#define QM_CL_EQCR_PI_CENA	0x3000
+#define QM_CL_EQCR_CI_CENA	0x3100
+#define QM_CL_DQRR_PI_CENA	0x3200
+#define QM_CL_DQRR_CI_CENA	0x3300
+#define QM_CL_MR_PI_CENA	0x3400
+#define QM_CL_MR_CI_CENA	0x3500
+#define QM_CL_CR		0x3800
+#define QM_CL_RR0		0x3900
+#define QM_CL_RR1		0x3940
+
+/*
+ * BTW, the drivers (and h/w programming model) already obtain the required
+ * synchronisation for portal accesses and data-dependencies. Use of barrier()s
+ * or other order-preserving primitives simply degrade performance. Hence the
+ * use of the __raw_*() interfaces, which simply ensure that the compiler treats
+ * the portal registers as volatile
+ */
 
-#include <asm/cacheflush.h>
-
-/* Last updated for v00.800 of the BG */
-
-/* Register offsets */
-#define REG_QCSP_LIO_CFG(n)	(0x0000 + ((n) * 0x10))
-#define REG_QCSP_IO_CFG(n)	(0x0004 + ((n) * 0x10))
-#define REG_QCSP_DD_CFG(n)	(0x000c + ((n) * 0x10))
-#define REG_DD_CFG		0x0200
-#define REG_DCP_CFG(n)		(0x0300 + ((n) * 0x10))
-#define REG_DCP_DD_CFG(n)	(0x0304 + ((n) * 0x10))
-#define REG_DCP_DLM_AVG(n)	(0x030c + ((n) * 0x10))
-#define REG_PFDR_FPC		0x0400
-#define REG_PFDR_FP_HEAD	0x0404
-#define REG_PFDR_FP_TAIL	0x0408
-#define REG_PFDR_FP_LWIT	0x0410
-#define REG_PFDR_CFG		0x0414
-#define REG_SFDR_CFG		0x0500
-#define REG_SFDR_IN_USE		0x0504
-#define REG_WQ_CS_CFG(n)	(0x0600 + ((n) * 0x04))
-#define REG_WQ_DEF_ENC_WQID	0x0630
-#define REG_WQ_SC_DD_CFG(n)	(0x640 + ((n) * 0x04))
-#define REG_WQ_PC_DD_CFG(n)	(0x680 + ((n) * 0x04))
-#define REG_WQ_DC0_DD_CFG(n)	(0x6c0 + ((n) * 0x04))
-#define REG_WQ_DC1_DD_CFG(n)	(0x700 + ((n) * 0x04))
-#define REG_WQ_DCn_DD_CFG(n)	(0x6c0 + ((n) * 0x40)) /* n=2,3 */
-#define REG_CM_CFG		0x0800
-#define REG_ECSR		0x0a00
-#define REG_ECIR		0x0a04
-#define REG_EADR		0x0a08
-#define REG_ECIR2		0x0a0c
-#define REG_EDATA(n)		(0x0a10 + ((n) * 0x04))
-#define REG_SBEC(n)		(0x0a80 + ((n) * 0x04))
-#define REG_MCR			0x0b00
-#define REG_MCP(n)		(0x0b04 + ((n) * 0x04))
-#define REG_MISC_CFG		0x0be0
-#define REG_HID_CFG		0x0bf0
-#define REG_IDLE_STAT		0x0bf4
-#define REG_IP_REV_1		0x0bf8
-#define REG_IP_REV_2		0x0bfc
-#define REG_FQD_BARE		0x0c00
-#define REG_PFDR_BARE		0x0c20
-#define REG_offset_BAR		0x0004	/* relative to REG_[FQD|PFDR]_BARE */
-#define REG_offset_AR		0x0010	/* relative to REG_[FQD|PFDR]_BARE */
-#define REG_QCSP_BARE		0x0c80
-#define REG_QCSP_BAR		0x0c84
-#define REG_CI_SCHED_CFG	0x0d00
-#define REG_SRCIDR		0x0d04
-#define REG_LIODNR		0x0d08
-#define REG_CI_RLM_AVG		0x0d14
-#define REG_ERR_ISR		0x0e00	/* + "enum qm_isr_reg" */
-#define REG_REV3_QCSP_LIO_CFG(n)	(0x1000 + ((n) * 0x10))
-#define REG_REV3_QCSP_IO_CFG(n)	(0x1004 + ((n) * 0x10))
-#define REG_REV3_QCSP_DD_CFG(n)	(0x100c + ((n) * 0x10))
-
-/* Assists for QMAN_MCR */
-#define MCR_INIT_PFDR		0x01000000
-#define MCR_get_rslt(v)		(u8)((v) >> 24)
-#define MCR_rslt_idle(r)	(!rslt || (rslt >= 0xf0))
-#define MCR_rslt_ok(r)		(rslt == 0xf0)
-#define MCR_rslt_eaccess(r)	(rslt == 0xf8)
-#define MCR_rslt_inval(r)	(rslt == 0xff)
-
-/* Corenet initiator settings. Stash request queues are 4-deep to match cores
-   ability to snarf. Stash priority is 3, other priorities are 2. */
-#define FSL_QMAN_CI_SCHED_CFG_SRCCIV   4
-#define FSL_QMAN_CI_SCHED_CFG_SRQ_W    3
-#define FSL_QMAN_CI_SCHED_CFG_RW_W     2
-#define FSL_QMAN_CI_SCHED_CFG_BMAN_W   2
-
-struct qman;
-
-/* Follows WQ_CS_CFG0-5 */
-enum qm_wq_class {
-	qm_wq_portal = 0,
-	qm_wq_pool = 1,
-	qm_wq_fman0 = 2,
-	qm_wq_fman1 = 3,
-	qm_wq_caam = 4,
-	qm_wq_pme = 5,
-	qm_wq_first = qm_wq_portal,
-	qm_wq_last = qm_wq_pme
+/* Cache-enabled ring access */
+#define qm_cl(base, idx)	((void *)base + ((idx) << 6))
+
+/*
+ * Portal modes.
+ *   Enum types;
+ *     pmode == production mode
+ *     cmode == consumption mode,
+ *     dmode == h/w dequeue mode.
+ *   Enum values use 3 letter codes. First letter matches the portal mode,
+ *   remaining two letters indicate;
+ *     ci == cache-inhibited portal register
+ *     ce == cache-enabled portal register
+ *     vb == in-band valid-bit (cache-enabled)
+ *     dc == DCA (Discrete Consumption Acknowledgment), DQRR-only
+ *   As for "enum qm_dqrr_dmode", it should be self-explanatory.
+ */
+enum qm_eqcr_pmode {		/* matches QCSP_CFG::EPM */
+	qm_eqcr_pci = 0,	/* PI index, cache-inhibited */
+	qm_eqcr_pce = 1,	/* PI index, cache-enabled */
+	qm_eqcr_pvb = 2		/* valid-bit */
+};
+enum qm_dqrr_dmode {		/* matches QCSP_CFG::DP */
+	qm_dqrr_dpush = 0,	/* SDQCR  + VDQCR */
+	qm_dqrr_dpull = 1	/* PDQCR */
+};
+enum qm_dqrr_pmode {		/* s/w-only */
+	qm_dqrr_pci,		/* reads DQRR_PI_CINH */
+	qm_dqrr_pce,		/* reads DQRR_PI_CENA */
+	qm_dqrr_pvb		/* reads valid-bit */
+};
+enum qm_dqrr_cmode {		/* matches QCSP_CFG::DCM */
+	qm_dqrr_cci = 0,	/* CI index, cache-inhibited */
+	qm_dqrr_cce = 1,	/* CI index, cache-enabled */
+	qm_dqrr_cdc = 2		/* Discrete Consumption Acknowledgment */
+};
+enum qm_mr_pmode {		/* s/w-only */
+	qm_mr_pci,		/* reads MR_PI_CINH */
+	qm_mr_pce,		/* reads MR_PI_CENA */
+	qm_mr_pvb		/* reads valid-bit */
+};
+enum qm_mr_cmode {		/* matches QCSP_CFG::MM */
+	qm_mr_cci = 0,		/* CI index, cache-inhibited */
+	qm_mr_cce = 1		/* CI index, cache-enabled */
 };
 
-/* Follows FQD_[BARE|BAR|AR] and PFDR_[BARE|BAR|AR] */
-enum qm_memory {
-	qm_memory_fqd,
-	qm_memory_pfdr
+/* --- Portal structures --- */
+
+#define QM_EQCR_SIZE		8
+#define QM_DQRR_SIZE		16
+#define QM_MR_SIZE		8
+
+/* "Enqueue Command" */
+struct qm_eqcr_entry {
+	u8 _ncw_verb; /* writes to this are non-coherent */
+	u8 dca;
+	__be16 seqnum;
+	u8 __reserved[4];
+	__be32 fqid;	/* 24-bit */
+	__be32 tag;
+	struct qm_fd fd;
+	u8 __reserved3[32];
+} __packed;
+#define QM_EQCR_VERB_VBIT		0x80
+#define QM_EQCR_VERB_CMD_MASK		0x61	/* but only one value; */
+#define QM_EQCR_VERB_CMD_ENQUEUE	0x01
+#define QM_EQCR_SEQNUM_NESN		0x8000	/* Advance NESN */
+#define QM_EQCR_SEQNUM_NLIS		0x4000	/* More fragments to come */
+#define QM_EQCR_SEQNUM_SEQMASK		0x3fff	/* sequence number goes here */
+
+struct qm_eqcr {
+	struct qm_eqcr_entry *ring, *cursor;
+	u8 ci, available, ithresh, vbit;
+#ifdef CONFIG_FSL_DPAA_CHECKING
+	u32 busy;
+	enum qm_eqcr_pmode pmode;
+#endif
 };
 
-/* Used by all error interrupt registers except 'inhibit' */
-#define QM_EIRQ_CIDE	0x20000000	/* Corenet Initiator Data Error */
-#define QM_EIRQ_CTDE	0x10000000	/* Corenet Target Data Error */
-#define QM_EIRQ_CITT	0x08000000	/* Corenet Invalid Target Transaction */
-#define QM_EIRQ_PLWI	0x04000000	/* PFDR Low Watermark */
-#define QM_EIRQ_MBEI	0x02000000	/* Multi-bit ECC Error */
-#define QM_EIRQ_SBEI	0x01000000	/* Single-bit ECC Error */
-#define QM_EIRQ_PEBI	0x00800000	/* PFDR Enqueues Blocked Interrupt */
-#define QM_EIRQ_IFSI	0x00020000	/* Invalid FQ Flow Control State */
-#define QM_EIRQ_ICVI	0x00010000	/* Invalid Command Verb */
-#define QM_EIRQ_IDDI	0x00000800	/* Invalid Dequeue (Direct-connect) */
-#define QM_EIRQ_IDFI	0x00000400	/* Invalid Dequeue FQ */
-#define QM_EIRQ_IDSI	0x00000200	/* Invalid Dequeue Source */
-#define QM_EIRQ_IDQI	0x00000100	/* Invalid Dequeue Queue */
-#define QM_EIRQ_IECE	0x00000010	/* Invalid Enqueue Configuration */
-#define QM_EIRQ_IEOI	0x00000008	/* Invalid Enqueue Overflow */
-#define QM_EIRQ_IESI	0x00000004	/* Invalid Enqueue State */
-#define QM_EIRQ_IECI	0x00000002	/* Invalid Enqueue Channel */
-#define QM_EIRQ_IEQI	0x00000001	/* Invalid Enqueue Queue */
-
-/* QMAN_ECIR valid error bit */
-#define PORTAL_ECSR_ERR	(QM_EIRQ_IEQI | QM_EIRQ_IESI | QM_EIRQ_IEOI | \
-				QM_EIRQ_IDQI | QM_EIRQ_IDSI | QM_EIRQ_IDFI | \
-				QM_EIRQ_IDDI | QM_EIRQ_ICVI | QM_EIRQ_IFSI)
-#define FQID_ECSR_ERR	(QM_EIRQ_IEQI | QM_EIRQ_IECI | QM_EIRQ_IESI | \
-			QM_EIRQ_IEOI | QM_EIRQ_IDQI | QM_EIRQ_IDFI | \
-			QM_EIRQ_IFSI)
-
-union qman_ecir {
-	u32 ecir_raw;
-	struct {
-		u32 __reserved:2;
-		u32 portal_type:1;
-		u32 portal_num:5;
-		u32 fqid:24;
-	} __packed info;
+struct qm_dqrr {
+	const struct qm_dqrr_entry *ring, *cursor;
+	u8 pi, ci, fill, ithresh, vbit;
+#ifdef CONFIG_FSL_DPAA_CHECKING
+	enum qm_dqrr_dmode dmode;
+	enum qm_dqrr_pmode pmode;
+	enum qm_dqrr_cmode cmode;
+#endif
 };
 
-union qman_ecir2 {
-	u32 ecir2_raw;
-	struct {
-		u32 portal_type:1;
-		u32 __reserved:21;
-		u32 portal_num:10;
-	} __packed info;
+struct qm_mr {
+	union qm_mr_entry *ring, *cursor;
+	u8 pi, ci, fill, ithresh, vbit;
+#ifdef CONFIG_FSL_DPAA_CHECKING
+	enum qm_mr_pmode pmode;
+	enum qm_mr_cmode cmode;
+#endif
 };
 
-union qman_eadr {
-	u32 eadr_raw;
-	struct {
-		u32 __reserved1:4;
-		u32 memid:4;
-		u32 __reserved2:12;
-		u32 eadr:12;
-	} __packed info;
-	struct {
-		u32 __reserved1:3;
-		u32 memid:5;
-		u32 __reserved:8;
-		u32 eadr:16;
-	} __packed info_rev3;
+/* MC (Management Command) command */
+/* "FQ" command layout */
+struct qm_mcc_fq {
+	u8 _ncw_verb;
+	u8 __reserved1[3];
+	__be32 fqid;	/* 24-bit */
+	u8 __reserved2[56];
+} __packed;
+
+/* "CGR" command layout */
+struct qm_mcc_cgr {
+	u8 _ncw_verb;
+	u8 __reserved1[30];
+	u8 cgid;
+	u8 __reserved2[32];
 };
 
-struct qman_hwerr_txt {
-	u32 mask;
-	const char *txt;
+#define QM_MCC_VERB_VBIT		0x80
+#define QM_MCC_VERB_MASK		0x7f	/* where the verb contains; */
+#define QM_MCC_VERB_INITFQ_PARKED	0x40
+#define QM_MCC_VERB_INITFQ_SCHED	0x41
+#define QM_MCC_VERB_QUERYFQ		0x44
+#define QM_MCC_VERB_QUERYFQ_NP		0x45	/* "non-programmable" fields */
+#define QM_MCC_VERB_QUERYWQ		0x46
+#define QM_MCC_VERB_QUERYWQ_DEDICATED	0x47
+#define QM_MCC_VERB_ALTER_SCHED		0x48	/* Schedule FQ */
+#define QM_MCC_VERB_ALTER_FE		0x49	/* Force Eligible FQ */
+#define QM_MCC_VERB_ALTER_RETIRE	0x4a	/* Retire FQ */
+#define QM_MCC_VERB_ALTER_OOS		0x4b	/* Take FQ out of service */
+#define QM_MCC_VERB_ALTER_FQXON		0x4d	/* FQ XON */
+#define QM_MCC_VERB_ALTER_FQXOFF	0x4e	/* FQ XOFF */
+#define QM_MCC_VERB_INITCGR		0x50
+#define QM_MCC_VERB_MODIFYCGR		0x51
+#define QM_MCC_VERB_CGRTESTWRITE	0x52
+#define QM_MCC_VERB_QUERYCGR		0x58
+#define QM_MCC_VERB_QUERYCONGESTION	0x59
+union qm_mc_command {
+	struct {
+		u8 _ncw_verb; /* writes to this are non-coherent */
+		u8 __reserved[63];
+	};
+	struct qm_mcc_initfq initfq;
+	struct qm_mcc_initcgr initcgr;
+	struct qm_mcc_fq fq;
+	struct qm_mcc_cgr cgr;
 };
 
-#define QMAN_HWE_TXT(a, b) { .mask = QM_EIRQ_##a, .txt = b }
-
-static const struct qman_hwerr_txt qman_hwerr_txts[] = {
-	QMAN_HWE_TXT(CIDE, "Corenet Initiator Data Error"),
-	QMAN_HWE_TXT(CTDE, "Corenet Target Data Error"),
-	QMAN_HWE_TXT(CITT, "Corenet Invalid Target Transaction"),
-	QMAN_HWE_TXT(PLWI, "PFDR Low Watermark"),
-	QMAN_HWE_TXT(MBEI, "Multi-bit ECC Error"),
-	QMAN_HWE_TXT(SBEI, "Single-bit ECC Error"),
-	QMAN_HWE_TXT(PEBI, "PFDR Enqueues Blocked Interrupt"),
-	QMAN_HWE_TXT(ICVI, "Invalid Command Verb"),
-	QMAN_HWE_TXT(IFSI, "Invalid Flow Control State"),
-	QMAN_HWE_TXT(IDDI, "Invalid Dequeue (Direct-connect)"),
-	QMAN_HWE_TXT(IDFI, "Invalid Dequeue FQ"),
-	QMAN_HWE_TXT(IDSI, "Invalid Dequeue Source"),
-	QMAN_HWE_TXT(IDQI, "Invalid Dequeue Queue"),
-	QMAN_HWE_TXT(IECE, "Invalid Enqueue Configuration"),
-	QMAN_HWE_TXT(IEOI, "Invalid Enqueue Overflow"),
-	QMAN_HWE_TXT(IESI, "Invalid Enqueue State"),
-	QMAN_HWE_TXT(IECI, "Invalid Enqueue Channel"),
-	QMAN_HWE_TXT(IEQI, "Invalid Enqueue Queue")
+/* MC (Management Command) result */
+/* "Query FQ" */
+struct qm_mcr_queryfq {
+	u8 verb;
+	u8 result;
+	u8 __reserved1[8];
+	struct qm_fqd fqd;	/* the FQD fields are here */
+	u8 __reserved2[30];
+} __packed;
+
+/* "Alter FQ State Commands" */
+struct qm_mcr_alterfq {
+	u8 verb;
+	u8 result;
+	u8 fqs;		/* Frame Queue Status */
+	u8 __reserved1[61];
+};
+#define QM_MCR_VERB_RRID		0x80
+#define QM_MCR_VERB_MASK		QM_MCC_VERB_MASK
+#define QM_MCR_VERB_INITFQ_PARKED	QM_MCC_VERB_INITFQ_PARKED
+#define QM_MCR_VERB_INITFQ_SCHED	QM_MCC_VERB_INITFQ_SCHED
+#define QM_MCR_VERB_QUERYFQ		QM_MCC_VERB_QUERYFQ
+#define QM_MCR_VERB_QUERYFQ_NP		QM_MCC_VERB_QUERYFQ_NP
+#define QM_MCR_VERB_QUERYWQ		QM_MCC_VERB_QUERYWQ
+#define QM_MCR_VERB_QUERYWQ_DEDICATED	QM_MCC_VERB_QUERYWQ_DEDICATED
+#define QM_MCR_VERB_ALTER_SCHED		QM_MCC_VERB_ALTER_SCHED
+#define QM_MCR_VERB_ALTER_FE		QM_MCC_VERB_ALTER_FE
+#define QM_MCR_VERB_ALTER_RETIRE	QM_MCC_VERB_ALTER_RETIRE
+#define QM_MCR_VERB_ALTER_OOS		QM_MCC_VERB_ALTER_OOS
+#define QM_MCR_RESULT_NULL		0x00
+#define QM_MCR_RESULT_OK		0xf0
+#define QM_MCR_RESULT_ERR_FQID		0xf1
+#define QM_MCR_RESULT_ERR_FQSTATE	0xf2
+#define QM_MCR_RESULT_ERR_NOTEMPTY	0xf3	/* OOS fails if FQ is !empty */
+#define QM_MCR_RESULT_ERR_BADCHANNEL	0xf4
+#define QM_MCR_RESULT_PENDING		0xf8
+#define QM_MCR_RESULT_ERR_BADCOMMAND	0xff
+#define QM_MCR_FQS_ORLPRESENT		0x02	/* ORL fragments to come */
+#define QM_MCR_FQS_NOTEMPTY		0x01	/* FQ has enqueued frames */
+#define QM_MCR_TIMEOUT			10000	/* us */
+union qm_mc_result {
+	struct {
+		u8 verb;
+		u8 result;
+		u8 __reserved1[62];
+	};
+	struct qm_mcr_queryfq queryfq;
+	struct qm_mcr_alterfq alterfq;
+	struct qm_mcr_querycgr querycgr;
+	struct qm_mcr_querycongestion querycongestion;
+	struct qm_mcr_querywq querywq;
+	struct qm_mcr_queryfq_np queryfq_np;
 };
-#define QMAN_HWE_COUNT (sizeof(qman_hwerr_txts)/sizeof(struct qman_hwerr_txt))
 
-struct qman_error_info_mdata {
-	u16 addr_mask;
-	u16 bits;
-	const char *txt;
+struct qm_mc {
+	union qm_mc_command *cr;
+	union qm_mc_result *rr;
+	u8 rridx, vbit;
+#ifdef CONFIG_FSL_DPAA_CHECKING
+	enum {
+		/* Can be _mc_start()ed */
+		qman_mc_idle,
+		/* Can be _mc_commit()ed or _mc_abort()ed */
+		qman_mc_user,
+		/* Can only be _mc_retry()ed */
+		qman_mc_hw
+	} state;
+#endif
 };
 
-#define QMAN_ERR_MDATA(a, b, c) { .addr_mask = a, .bits = b, .txt = c}
-static const struct qman_error_info_mdata error_mdata[] = {
-	QMAN_ERR_MDATA(0x01FF, 24, "FQD cache tag memory 0"),
-	QMAN_ERR_MDATA(0x01FF, 24, "FQD cache tag memory 1"),
-	QMAN_ERR_MDATA(0x01FF, 24, "FQD cache tag memory 2"),
-	QMAN_ERR_MDATA(0x01FF, 24, "FQD cache tag memory 3"),
-	QMAN_ERR_MDATA(0x0FFF, 512, "FQD cache memory"),
-	QMAN_ERR_MDATA(0x07FF, 128, "SFDR memory"),
-	QMAN_ERR_MDATA(0x01FF, 72, "WQ context memory"),
-	QMAN_ERR_MDATA(0x00FF, 240, "CGR memory"),
-	QMAN_ERR_MDATA(0x00FF, 302, "Internal Order Restoration List memory"),
-	QMAN_ERR_MDATA(0x01FF, 256, "SW portal ring memory"),
+struct qm_addr {
+	void __iomem *ce;	/* cache-enabled */
+	void __iomem *ci;	/* cache-inhibited */
 };
-#define QMAN_ERR_MDATA_COUNT \
-	(sizeof(error_mdata)/sizeof(struct qman_error_info_mdata))
 
-/* Add this in Kconfig */
-#define QMAN_ERRS_TO_UNENABLE (QM_EIRQ_PLWI | QM_EIRQ_PEBI)
+struct qm_portal {
+	/*
+	 * In the non-CONFIG_FSL_DPAA_CHECKING case, the following stuff up to
+	 * and including 'mc' fits within a cacheline (yay!). The 'config' part
+	 * is setup-only, so isn't a cause for a concern. In other words, don't
+	 * rearrange this structure on a whim, there be dragons ...
+	 */
+	struct qm_addr addr;
+	struct qm_eqcr eqcr;
+	struct qm_dqrr dqrr;
+	struct qm_mr mr;
+	struct qm_mc mc;
+} ____cacheline_aligned;
+
+/* Cache-inhibited register access. */
+static inline u32 qm_in(struct qm_portal *p, u32 offset)
+{
+	return be32_to_cpu(__raw_readl(p->addr.ci + offset));
+}
 
-/**
- * qm_err_isr_<reg>_<verb> - Manipulate global interrupt registers
- * @v: for accessors that write values, this is the 32-bit value
- *
- * Manipulates QMAN_ERR_ISR, QMAN_ERR_IER, QMAN_ERR_ISDR, QMAN_ERR_IIR. All
- * manipulations except qm_err_isr_[un]inhibit() use 32-bit masks composed of
- * the QM_EIRQ_*** definitions. Note that "qm_err_isr_enable_write" means
- * "write the enable register" rather than "enable the write register"!
- */
-#define qm_err_isr_status_read(qm)	\
-		__qm_err_isr_read(qm, qm_isr_status)
-#define qm_err_isr_status_clear(qm, m)	\
-		__qm_err_isr_write(qm, qm_isr_status, m)
-#define qm_err_isr_enable_read(qm)	\
-		__qm_err_isr_read(qm, qm_isr_enable)
-#define qm_err_isr_enable_write(qm, v)	\
-		__qm_err_isr_write(qm, qm_isr_enable, v)
-#define qm_err_isr_disable_read(qm)	\
-		__qm_err_isr_read(qm, qm_isr_disable)
-#define qm_err_isr_disable_write(qm, v)	\
-		__qm_err_isr_write(qm, qm_isr_disable, v)
-#define qm_err_isr_inhibit(qm)		\
-		__qm_err_isr_write(qm, qm_isr_inhibit, 1)
-#define qm_err_isr_uninhibit(qm)	\
-		__qm_err_isr_write(qm, qm_isr_inhibit, 0)
+static inline void qm_out(struct qm_portal *p, u32 offset, u32 val)
+{
+	__raw_writel(cpu_to_be32(val), p->addr.ci + offset);
+}
 
-/*
- * TODO: unimplemented registers
- *
- * Keeping a list here of QMan registers I have not yet covered;
- * QCSP_DD_IHRSR, QCSP_DD_IHRFR, QCSP_DD_HASR,
- * DCP_DD_IHRSR, DCP_DD_IHRFR, DCP_DD_HASR, CM_CFG,
- * QMAN_EECC, QMAN_SBET, QMAN_EINJ, QMAN_SBEC0-12
- */
+/* Cache Enabled Portal Access */
+static inline void qm_cl_invalidate(struct qm_portal *p, u32 offset)
+{
+	dpaa_invalidate(p->addr.ce + offset);
+}
 
-/* Encapsulate "struct qman *" as a cast of the register space address. */
+static inline void qm_cl_touch_ro(struct qm_portal *p, u32 offset)
+{
+	dpaa_touch_ro(p->addr.ce + offset);
+}
 
-static struct qman *qm_create(void *regs)
+static inline u32 qm_ce_in(struct qm_portal *p, u32 offset)
 {
-	return (struct qman *)regs;
+	return be32_to_cpu(__raw_readl(p->addr.ce + offset));
 }
 
-static inline u32 __qm_in(struct qman *qm, u32 offset)
+/* --- EQCR API --- */
+
+#define EQCR_SHIFT	ilog2(sizeof(struct qm_eqcr_entry))
+#define EQCR_CARRY	(uintptr_t)(QM_EQCR_SIZE << EQCR_SHIFT)
+
+/* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */
+static struct qm_eqcr_entry *eqcr_carryclear(struct qm_eqcr_entry *p)
 {
-	return ioread32be((void *)qm + offset);
+	uintptr_t addr = (uintptr_t)p;
+
+	addr &= ~EQCR_CARRY;
+
+	return (struct qm_eqcr_entry *)addr;
 }
-static inline void __qm_out(struct qman *qm, u32 offset, u32 val)
+
+/* Bit-wise logic to convert a ring pointer to a ring index */
+static int eqcr_ptr2idx(struct qm_eqcr_entry *e)
 {
-	iowrite32be(val, (void *)qm + offset);
+	return ((uintptr_t)e >> EQCR_SHIFT) & (QM_EQCR_SIZE - 1);
 }
-#define qm_in(reg)		__qm_in(qm, REG_##reg)
-#define qm_out(reg, val)	__qm_out(qm, REG_##reg, val)
 
-static u32 __qm_err_isr_read(struct qman *qm, enum qm_isr_reg n)
+/* Increment the 'cursor' ring pointer, taking 'vbit' into account */
+static inline void eqcr_inc(struct qm_eqcr *eqcr)
 {
-	return __qm_in(qm, REG_ERR_ISR + (n << 2));
+	/* increment to the next EQCR pointer and handle overflow and 'vbit' */
+	struct qm_eqcr_entry *partial = eqcr->cursor + 1;
+
+	eqcr->cursor = eqcr_carryclear(partial);
+	if (partial != eqcr->cursor)
+		eqcr->vbit ^= QM_EQCR_VERB_VBIT;
 }
 
-static void __qm_err_isr_write(struct qman *qm, enum qm_isr_reg n, u32 val)
+static inline int qm_eqcr_init(struct qm_portal *portal,
+				enum qm_eqcr_pmode pmode,
+				unsigned int eq_stash_thresh,
+				int eq_stash_prio)
 {
-	__qm_out(qm, REG_ERR_ISR + (n << 2), val);
+	struct qm_eqcr *eqcr = &portal->eqcr;
+	u32 cfg;
+	u8 pi;
+
+	eqcr->ring = portal->addr.ce + QM_CL_EQCR;
+	eqcr->ci = qm_in(portal, QM_REG_EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
+	qm_cl_invalidate(portal, QM_CL_EQCR_CI_CENA);
+	pi = qm_in(portal, QM_REG_EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
+	eqcr->cursor = eqcr->ring + pi;
+	eqcr->vbit = (qm_in(portal, QM_REG_EQCR_PI_CINH) & QM_EQCR_SIZE) ?
+		     QM_EQCR_VERB_VBIT : 0;
+	eqcr->available = QM_EQCR_SIZE - 1 -
+			  dpaa_cyc_diff(QM_EQCR_SIZE, eqcr->ci, pi);
+	eqcr->ithresh = qm_in(portal, QM_REG_EQCR_ITR);
+#ifdef CONFIG_FSL_DPAA_CHECKING
+	eqcr->busy = 0;
+	eqcr->pmode = pmode;
+#endif
+	cfg = (qm_in(portal, QM_REG_CFG) & 0x00ffffff) |
+	      (eq_stash_thresh << 28) | /* QCSP_CFG: EST */
+	      (eq_stash_prio << 26) | /* QCSP_CFG: EP */
+	      ((pmode & 0x3) << 24); /* QCSP_CFG::EPM */
+	qm_out(portal, QM_REG_CFG, cfg);
+	return 0;
 }
 
-static void qm_set_dc(struct qman *qm, enum qm_dc_portal portal,
-			int ed, u8 sernd)
+static inline unsigned int qm_eqcr_get_ci_stashing(struct qm_portal *portal)
 {
-	DPA_ASSERT(!ed || (portal == qm_dc_portal_fman0) ||
-			(portal == qm_dc_portal_fman1));
-	if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
-		qm_out(DCP_CFG(portal), (ed ? 0x1000 : 0) | (sernd & 0x3ff));
-	else
-		qm_out(DCP_CFG(portal), (ed ? 0x100 : 0) | (sernd & 0x1f));
+	return (qm_in(portal, QM_REG_CFG) >> 28) & 0x7;
 }
 
-static void qm_set_wq_scheduling(struct qman *qm, enum qm_wq_class wq_class,
-			u8 cs_elev, u8 csw2, u8 csw3, u8 csw4, u8 csw5,
-			u8 csw6, u8 csw7)
+static inline void qm_eqcr_finish(struct qm_portal *portal)
 {
-	qm_out(WQ_CS_CFG(wq_class), ((cs_elev & 0xff) << 24) |
-		((csw2 & 0x7) << 20) | ((csw3 & 0x7) << 16) |
-		((csw4 & 0x7) << 12) | ((csw5 & 0x7) << 8) |
-		((csw6 & 0x7) << 4) | (csw7 & 0x7));
+	struct qm_eqcr *eqcr = &portal->eqcr;
+	u8 pi = qm_in(portal, QM_REG_EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
+	u8 ci = qm_in(portal, QM_REG_EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
+
+	DPAA_ASSERT(!eqcr->busy);
+	if (pi != eqcr_ptr2idx(eqcr->cursor))
+		pr_crit("losing uncommitted EQCR entries\n");
+	if (ci != eqcr->ci)
+		pr_crit("missing existing EQCR completions\n");
+	if (eqcr->ci != eqcr_ptr2idx(eqcr->cursor))
+		pr_crit("EQCR destroyed unquiesced\n");
 }
 
-static void qm_set_hid(struct qman *qm)
+static inline struct qm_eqcr_entry *qm_eqcr_start_no_stash(struct qm_portal
+								 *portal)
 {
-	qm_out(HID_CFG, 0);
+	struct qm_eqcr *eqcr = &portal->eqcr;
+
+	DPAA_ASSERT(!eqcr->busy);
+	if (!eqcr->available)
+		return NULL;
+
+#ifdef CONFIG_FSL_DPAA_CHECKING
+	eqcr->busy = 1;
+#endif
+	dpaa_zero(eqcr->cursor);
+	return eqcr->cursor;
 }
 
-static void qm_set_corenet_initiator(struct qman *qm)
+static inline struct qm_eqcr_entry *qm_eqcr_start_stash(struct qm_portal
+								*portal)
 {
-	qm_out(CI_SCHED_CFG,
-	       0x80000000 | /* write srcciv enable */
-	       (FSL_QMAN_CI_SCHED_CFG_SRCCIV << 24) |
-	       (FSL_QMAN_CI_SCHED_CFG_SRQ_W << 8) |
-	       (FSL_QMAN_CI_SCHED_CFG_RW_W << 4) |
-	       FSL_QMAN_CI_SCHED_CFG_BMAN_W);
+	struct qm_eqcr *eqcr = &portal->eqcr;
+	u8 diff, old_ci;
+
+	DPAA_ASSERT(!eqcr->busy);
+	if (!eqcr->available) {
+		old_ci = eqcr->ci;
+		eqcr->ci = qm_ce_in(portal, QM_CL_EQCR_CI_CENA) &
+			   (QM_EQCR_SIZE - 1);
+		diff = dpaa_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
+		eqcr->available += diff;
+		if (!diff)
+			return NULL;
+	}
+#ifdef CONFIG_FSL_DPAA_CHECKING
+	eqcr->busy = 1;
+#endif
+	dpaa_zero(eqcr->cursor);
+	return eqcr->cursor;
 }
 
-static void qm_get_version(struct qman *qm, u16 *id, u8 *major, u8 *minor)
+static inline void eqcr_commit_checks(struct qm_eqcr *eqcr)
 {
-	u32 v = qm_in(IP_REV_1);
-	*id = (v >> 16);
-	*major = (v >> 8) & 0xff;
-	*minor = v & 0xff;
+	DPAA_ASSERT(eqcr->busy);
+	DPAA_ASSERT(!(be32_to_cpu(eqcr->cursor->fqid) & ~QM_FQID_MASK));
+	DPAA_ASSERT(eqcr->available >= 1);
 }
 
-static void qm_set_memory(struct qman *qm, enum qm_memory memory, u64 ba,
-			int enable, int prio, int stash, u32 size)
+static inline void qm_eqcr_pvb_commit(struct qm_portal *portal, u8 myverb)
 {
-	u32 offset = (memory == qm_memory_fqd) ? REG_FQD_BARE : REG_PFDR_BARE;
-	u32 exp = ilog2(size);
-	/* choke if size isn't within range */
-	DPA_ASSERT((size >= 4096) && (size <= 1073741824) &&
-			is_power_of_2(size));
-	/* choke if 'ba' has lower-alignment than 'size' */
-	DPA_ASSERT(!(ba & (size - 1)));
-	__qm_out(qm, offset, upper_32_bits(ba));
-	__qm_out(qm, offset + REG_offset_BAR, lower_32_bits(ba));
-	__qm_out(qm, offset + REG_offset_AR,
-		(enable ? 0x80000000 : 0) |
-		(prio ? 0x40000000 : 0) |
-		(stash ? 0x20000000 : 0) |
-		(exp - 1));
+	struct qm_eqcr *eqcr = &portal->eqcr;
+	struct qm_eqcr_entry *eqcursor;
+
+	eqcr_commit_checks(eqcr);
+	DPAA_ASSERT(eqcr->pmode == qm_eqcr_pvb);
+	dma_wmb();
+	eqcursor = eqcr->cursor;
+	eqcursor->_ncw_verb = myverb | eqcr->vbit;
+	dpaa_flush(eqcursor);
+	eqcr_inc(eqcr);
+	eqcr->available--;
+#ifdef CONFIG_FSL_DPAA_CHECKING
+	eqcr->busy = 0;
+#endif
 }
 
-static void qm_set_pfdr_threshold(struct qman *qm, u32 th, u8 k)
+static inline void qm_eqcr_cce_prefetch(struct qm_portal *portal)
 {
-	qm_out(PFDR_FP_LWIT, th & 0xffffff);
-	qm_out(PFDR_CFG, k);
+	qm_cl_touch_ro(portal, QM_CL_EQCR_CI_CENA);
 }
 
-static void qm_set_sfdr_threshold(struct qman *qm, u16 th)
+static inline u8 qm_eqcr_cce_update(struct qm_portal *portal)
 {
-	qm_out(SFDR_CFG, th & 0x3ff);
+	struct qm_eqcr *eqcr = &portal->eqcr;
+	u8 diff, old_ci = eqcr->ci;
+
+	eqcr->ci = qm_ce_in(portal, QM_CL_EQCR_CI_CENA) & (QM_EQCR_SIZE - 1);
+	qm_cl_invalidate(portal, QM_CL_EQCR_CI_CENA);
+	diff = dpaa_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
+	eqcr->available += diff;
+	return diff;
 }
 
-static int qm_init_pfdr(struct qman *qm, u32 pfdr_start, u32 num)
+static inline void qm_eqcr_set_ithresh(struct qm_portal *portal, u8 ithresh)
 {
-	u8 rslt = MCR_get_rslt(qm_in(MCR));
+	struct qm_eqcr *eqcr = &portal->eqcr;
+
+	eqcr->ithresh = ithresh;
+	qm_out(portal, QM_REG_EQCR_ITR, ithresh);
+}
 
-	DPA_ASSERT(pfdr_start && !(pfdr_start & 7) && !(num & 7) && num);
-	/* Make sure the command interface is 'idle' */
-	if (!MCR_rslt_idle(rslt))
-		panic("QMAN_MCR isn't idle");
+static inline u8 qm_eqcr_get_avail(struct qm_portal *portal)
+{
+	struct qm_eqcr *eqcr = &portal->eqcr;
 
-	/* Write the MCR command params then the verb */
-	qm_out(MCP(0), pfdr_start);
-	/* TODO: remove this - it's a workaround for a model bug that is
-	 * corrected in more recent versions. We use the workaround until
-	 * everyone has upgraded. */
-	qm_out(MCP(1), (pfdr_start + num - 16));
-	lwsync();
-	qm_out(MCR, MCR_INIT_PFDR);
-	/* Poll for the result */
-	do {
-		rslt = MCR_get_rslt(qm_in(MCR));
-	} while (!MCR_rslt_idle(rslt));
-	if (MCR_rslt_ok(rslt))
-		return 0;
-	if (MCR_rslt_eaccess(rslt))
-		return -EACCES;
-	if (MCR_rslt_inval(rslt))
-		return -EINVAL;
-	pr_crit("Unexpected result from MCR_INIT_PFDR: %02x\n", rslt);
-	return -ENODEV;
+	return eqcr->available;
 }
 
-/*****************/
-/* Config driver */
-/*****************/
+static inline u8 qm_eqcr_get_fill(struct qm_portal *portal)
+{
+	struct qm_eqcr *eqcr = &portal->eqcr;
 
-/* We support only one of these */
-static struct qman *qm;
-#ifndef __rtems__
-static struct device_node *qm_node;
-#endif /* __rtems__ */
+	return QM_EQCR_SIZE - 1 - eqcr->available;
+}
 
-/* And this state belongs to 'qm'. It is set during fsl_qman_init(), but used
- * during qman_init_ccsr(). */
-#ifndef __rtems__
-static dma_addr_t fqd_a, pfdr_a;
-static size_t fqd_sz, pfdr_sz;
+/* --- DQRR API --- */
 
-static int qman_fqd(struct reserved_mem *rmem)
+#define DQRR_SHIFT	ilog2(sizeof(struct qm_dqrr_entry))
+#define DQRR_CARRY	(uintptr_t)(QM_DQRR_SIZE << DQRR_SHIFT)
+
+static const struct qm_dqrr_entry *dqrr_carryclear(
+					const struct qm_dqrr_entry *p)
 {
-	fqd_a = rmem->base;
-	fqd_sz = rmem->size;
+	uintptr_t addr = (uintptr_t)p;
 
-	WARN_ON(!(fqd_a && fqd_sz));
+	addr &= ~DQRR_CARRY;
 
-	return 0;
+	return (const struct qm_dqrr_entry *)addr;
 }
-RESERVEDMEM_OF_DECLARE(qman_fqd, "fsl,qman-fqd", qman_fqd);
-#else /* __rtems__ */
-static DPAA_NOCACHENOLOAD_ALIGNED_REGION(fqd, 4194304);
-#define fqd_a ((uintptr_t)&fqd[0])
-#define fqd_sz sizeof(fqd)
-static DPAA_NOCACHENOLOAD_ALIGNED_REGION(pfdr, 33554432);
-#define pfdr_a ((uintptr_t)&pfdr[0])
-#define pfdr_sz sizeof(pfdr)
-#endif /* __rtems__ */
 
-size_t qman_fqd_size(void)
+static inline int dqrr_ptr2idx(const struct qm_dqrr_entry *e)
 {
-	return fqd_sz;
+	return ((uintptr_t)e >> DQRR_SHIFT) & (QM_DQRR_SIZE - 1);
 }
 
-#ifndef __rtems__
-static int qman_pfdr(struct reserved_mem *rmem)
+static const struct qm_dqrr_entry *dqrr_inc(const struct qm_dqrr_entry *e)
 {
-	pfdr_a = rmem->base;
-	pfdr_sz = rmem->size;
+	return dqrr_carryclear(e + 1);
+}
 
-	WARN_ON(!(pfdr_a && pfdr_sz));
+static inline void qm_dqrr_set_maxfill(struct qm_portal *portal, u8 mf)
+{
+	qm_out(portal, QM_REG_CFG, (qm_in(portal, QM_REG_CFG) & 0xff0fffff) |
+				   ((mf & (QM_DQRR_SIZE - 1)) << 20));
+}
 
+static inline int qm_dqrr_init(struct qm_portal *portal,
+			       const struct qm_portal_config *config,
+			       enum qm_dqrr_dmode dmode,
+			       enum qm_dqrr_pmode pmode,
+			       enum qm_dqrr_cmode cmode, u8 max_fill)
+{
+	struct qm_dqrr *dqrr = &portal->dqrr;
+	u32 cfg;
+
+	/* Make sure the DQRR will be idle when we enable */
+	qm_out(portal, QM_REG_DQRR_SDQCR, 0);
+	qm_out(portal, QM_REG_DQRR_VDQCR, 0);
+	qm_out(portal, QM_REG_DQRR_PDQCR, 0);
+	dqrr->ring = portal->addr.ce + QM_CL_DQRR;
+	dqrr->pi = qm_in(portal, QM_REG_DQRR_PI_CINH) & (QM_DQRR_SIZE - 1);
+	dqrr->ci = qm_in(portal, QM_REG_DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);
+	dqrr->cursor = dqrr->ring + dqrr->ci;
+	dqrr->fill = dpaa_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi);
+	dqrr->vbit = (qm_in(portal, QM_REG_DQRR_PI_CINH) & QM_DQRR_SIZE) ?
+			QM_DQRR_VERB_VBIT : 0;
+	dqrr->ithresh = qm_in(portal, QM_REG_DQRR_ITR);
+#ifdef CONFIG_FSL_DPAA_CHECKING
+	dqrr->dmode = dmode;
+	dqrr->pmode = pmode;
+	dqrr->cmode = cmode;
+#endif
+	/* Invalidate every ring entry before beginning */
+	for (cfg = 0; cfg < QM_DQRR_SIZE; cfg++)
+		dpaa_invalidate(qm_cl(dqrr->ring, cfg));
+	cfg = (qm_in(portal, QM_REG_CFG) & 0xff000f00) |
+		((max_fill & (QM_DQRR_SIZE - 1)) << 20) | /* DQRR_MF */
+		((dmode & 1) << 18) |			/* DP */
+		((cmode & 3) << 16) |			/* DCM */
+		0xa0 |					/* RE+SE */
+		(0 ? 0x40 : 0) |			/* Ignore RP */
+		(0 ? 0x10 : 0);				/* Ignore SP */
+	qm_out(portal, QM_REG_CFG, cfg);
+	qm_dqrr_set_maxfill(portal, max_fill);
 	return 0;
 }
-RESERVEDMEM_OF_DECLARE(qman_fbpr, "fsl,qman-pfdr", qman_pfdr);
 
-/* Parse the <name> property to extract the memory location and size and
- * memblock_reserve() it. If it isn't supplied, memblock_alloc() the default
- * size. Also flush this memory range from data cache so that QMAN originated
- * transactions for this memory region could be marked non-coherent.
- */
-static __init int parse_mem_property(struct device_node *node,
-				     dma_addr_t *addr, size_t *sz, int zero)
+static inline void qm_dqrr_finish(struct qm_portal *portal)
 {
-	int ret;
+#ifdef CONFIG_FSL_DPAA_CHECKING
+	struct qm_dqrr *dqrr = &portal->dqrr;
+
+	if (dqrr->cmode != qm_dqrr_cdc &&
+	    dqrr->ci != dqrr_ptr2idx(dqrr->cursor))
+		pr_crit("Ignoring completed DQRR entries\n");
+#endif
+}
+
+static inline const struct qm_dqrr_entry *qm_dqrr_current(
+						struct qm_portal *portal)
+{
+	struct qm_dqrr *dqrr = &portal->dqrr;
+
+	if (!dqrr->fill)
+		return NULL;
+	return dqrr->cursor;
+}
+
+static inline u8 qm_dqrr_next(struct qm_portal *portal)
+{
+	struct qm_dqrr *dqrr = &portal->dqrr;
+
+	DPAA_ASSERT(dqrr->fill);
+	dqrr->cursor = dqrr_inc(dqrr->cursor);
+	return --dqrr->fill;
+}
 
-	/* If using a "zero-pma", don't try to zero it, even if you asked */
-	if (zero && of_find_property(node, "zero-pma", &ret)) {
-		pr_info("  it's a 'zero-pma', not zeroing from s/w\n");
-		zero = 0;
+static inline void qm_dqrr_pvb_update(struct qm_portal *portal)
+{
+	struct qm_dqrr *dqrr = &portal->dqrr;
+	struct qm_dqrr_entry *res = qm_cl(dqrr->ring, dqrr->pi);
+
+	DPAA_ASSERT(dqrr->pmode == qm_dqrr_pvb);
+#ifndef CONFIG_FSL_PAMU
+	/*
+	 * If PAMU is not available we need to invalidate the cache.
+	 * When PAMU is available the cache is updated by stash
+	 */
+	dpaa_invalidate_touch_ro(res);
+#endif
+	/*
+	 *  when accessing 'verb', use __raw_readb() to ensure that compiler
+	 * inlining doesn't try to optimise out "excess reads".
+	 */
+	if ((__raw_readb(&res->verb) & QM_DQRR_VERB_VBIT) == dqrr->vbit) {
+		dqrr->pi = (dqrr->pi + 1) & (QM_DQRR_SIZE - 1);
+		if (!dqrr->pi)
+			dqrr->vbit ^= QM_DQRR_VERB_VBIT;
+		dqrr->fill++;
 	}
+}
+
+static inline void qm_dqrr_cdc_consume_1ptr(struct qm_portal *portal,
+					const struct qm_dqrr_entry *dq,
+					int park)
+{
+	__maybe_unused struct qm_dqrr *dqrr = &portal->dqrr;
+	int idx = dqrr_ptr2idx(dq);
+
+	DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
+	DPAA_ASSERT((dqrr->ring + idx) == dq);
+	DPAA_ASSERT(idx < QM_DQRR_SIZE);
+	qm_out(portal, QM_REG_DQRR_DCAP, (0 << 8) | /* DQRR_DCAP::S */
+	       ((park ? 1 : 0) << 6) |		    /* DQRR_DCAP::PK */
+	       idx);				    /* DQRR_DCAP::DCAP_CI */
+}
+
+static inline void qm_dqrr_cdc_consume_n(struct qm_portal *portal, u32 bitmask)
+{
+	__maybe_unused struct qm_dqrr *dqrr = &portal->dqrr;
+
+	DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
+	qm_out(portal, QM_REG_DQRR_DCAP, (1 << 8) | /* DQRR_DCAP::S */
+	       (bitmask << 16));		    /* DQRR_DCAP::DCAP_CI */
+}
+
+static inline void qm_dqrr_sdqcr_set(struct qm_portal *portal, u32 sdqcr)
+{
+	qm_out(portal, QM_REG_DQRR_SDQCR, sdqcr);
+}
+
+static inline void qm_dqrr_vdqcr_set(struct qm_portal *portal, u32 vdqcr)
+{
+	qm_out(portal, QM_REG_DQRR_VDQCR, vdqcr);
+}
+
+static inline void qm_dqrr_set_ithresh(struct qm_portal *portal, u8 ithresh)
+{
+	qm_out(portal, QM_REG_DQRR_ITR, ithresh);
+}
+
+/* --- MR API --- */
+
+#define MR_SHIFT	ilog2(sizeof(union qm_mr_entry))
+#define MR_CARRY	(uintptr_t)(QM_MR_SIZE << MR_SHIFT)
+
+static union qm_mr_entry *mr_carryclear(union qm_mr_entry *p)
+{
+	uintptr_t addr = (uintptr_t)p;
+
+	addr &= ~MR_CARRY;
+
+	return (union qm_mr_entry *)addr;
+}
+
+static inline int mr_ptr2idx(const union qm_mr_entry *e)
+{
+	return ((uintptr_t)e >> MR_SHIFT) & (QM_MR_SIZE - 1);
+}
+
+static inline union qm_mr_entry *mr_inc(union qm_mr_entry *e)
+{
+	return mr_carryclear(e + 1);
+}
+
+static inline int qm_mr_init(struct qm_portal *portal, enum qm_mr_pmode pmode,
+			     enum qm_mr_cmode cmode)
+{
+	struct qm_mr *mr = &portal->mr;
+	u32 cfg;
+
+	mr->ring = portal->addr.ce + QM_CL_MR;
+	mr->pi = qm_in(portal, QM_REG_MR_PI_CINH) & (QM_MR_SIZE - 1);
+	mr->ci = qm_in(portal, QM_REG_MR_CI_CINH) & (QM_MR_SIZE - 1);
+	mr->cursor = mr->ring + mr->ci;
+	mr->fill = dpaa_cyc_diff(QM_MR_SIZE, mr->ci, mr->pi);
+	mr->vbit = (qm_in(portal, QM_REG_MR_PI_CINH) & QM_MR_SIZE)
+		? QM_MR_VERB_VBIT : 0;
+	mr->ithresh = qm_in(portal, QM_REG_MR_ITR);
+#ifdef CONFIG_FSL_DPAA_CHECKING
+	mr->pmode = pmode;
+	mr->cmode = cmode;
+#endif
+	cfg = (qm_in(portal, QM_REG_CFG) & 0xfffff0ff) |
+	      ((cmode & 1) << 8);	/* QCSP_CFG:MM */
+	qm_out(portal, QM_REG_CFG, cfg);
+	return 0;
+}
+
+static inline void qm_mr_finish(struct qm_portal *portal)
+{
+	struct qm_mr *mr = &portal->mr;
+
+	if (mr->ci != mr_ptr2idx(mr->cursor))
+		pr_crit("Ignoring completed MR entries\n");
+}
+
+static inline const union qm_mr_entry *qm_mr_current(struct qm_portal *portal)
+{
+	struct qm_mr *mr = &portal->mr;
+
+	if (!mr->fill)
+		return NULL;
+	return mr->cursor;
+}
 
-	if (zero) {
-		/* map as cacheable, non-guarded */
-		void __iomem *tmpp = ioremap_prot(*addr, *sz, 0);
+static inline int qm_mr_next(struct qm_portal *portal)
+{
+	struct qm_mr *mr = &portal->mr;
 
-		memset_io(tmpp, 0, *sz);
-		flush_dcache_range((unsigned long)tmpp,
-				   (unsigned long)tmpp + *sz);
-		iounmap(tmpp);
+	DPAA_ASSERT(mr->fill);
+	mr->cursor = mr_inc(mr->cursor);
+	return --mr->fill;
+}
+
+static inline void qm_mr_pvb_update(struct qm_portal *portal)
+{
+	struct qm_mr *mr = &portal->mr;
+	union qm_mr_entry *res = qm_cl(mr->ring, mr->pi);
+
+	DPAA_ASSERT(mr->pmode == qm_mr_pvb);
+	/*
+	 *  when accessing 'verb', use __raw_readb() to ensure that compiler
+	 * inlining doesn't try to optimise out "excess reads".
+	 */
+	if ((__raw_readb(&res->verb) & QM_MR_VERB_VBIT) == mr->vbit) {
+		mr->pi = (mr->pi + 1) & (QM_MR_SIZE - 1);
+		if (!mr->pi)
+			mr->vbit ^= QM_MR_VERB_VBIT;
+		mr->fill++;
+		res = mr_inc(res);
 	}
+	dpaa_invalidate_touch_ro(res);
+}
+
+static inline void qm_mr_cci_consume(struct qm_portal *portal, u8 num)
+{
+	struct qm_mr *mr = &portal->mr;
+
+	DPAA_ASSERT(mr->cmode == qm_mr_cci);
+	mr->ci = (mr->ci + num) & (QM_MR_SIZE - 1);
+	qm_out(portal, QM_REG_MR_CI_CINH, mr->ci);
+}
+
+static inline void qm_mr_cci_consume_to_current(struct qm_portal *portal)
+{
+	struct qm_mr *mr = &portal->mr;
+
+	DPAA_ASSERT(mr->cmode == qm_mr_cci);
+	mr->ci = mr_ptr2idx(mr->cursor);
+	qm_out(portal, QM_REG_MR_CI_CINH, mr->ci);
+}
+
+static inline void qm_mr_set_ithresh(struct qm_portal *portal, u8 ithresh)
+{
+	qm_out(portal, QM_REG_MR_ITR, ithresh);
+}
+
+/* --- Management command API --- */
 
+static inline int qm_mc_init(struct qm_portal *portal)
+{
+	struct qm_mc *mc = &portal->mc;
+
+	mc->cr = portal->addr.ce + QM_CL_CR;
+	mc->rr = portal->addr.ce + QM_CL_RR0;
+	mc->rridx = (__raw_readb(&mc->cr->_ncw_verb) & QM_MCC_VERB_VBIT)
+		    ? 0 : 1;
+	mc->vbit = mc->rridx ? QM_MCC_VERB_VBIT : 0;
+#ifdef CONFIG_FSL_DPAA_CHECKING
+	mc->state = qman_mc_idle;
+#endif
 	return 0;
 }
-#else /* __rtems__ */
-#include <bsp/qoriq.h>
-#endif /* __rtems__ */
 
-/* TODO:
- * - there is obviously no handling of errors,
- * - the calls to qm_set_memory() hard-code the priority and CPC-stashing for
- *   both memory resources to zero.
- */
-static int __init fsl_qman_init(struct device_node *node)
+static inline void qm_mc_finish(struct qm_portal *portal)
+{
+#ifdef CONFIG_FSL_DPAA_CHECKING
+	struct qm_mc *mc = &portal->mc;
+
+	DPAA_ASSERT(mc->state == qman_mc_idle);
+	if (mc->state != qman_mc_idle)
+		pr_crit("Losing incomplete MC command\n");
+#endif
+}
+
+static inline union qm_mc_command *qm_mc_start(struct qm_portal *portal)
+{
+	struct qm_mc *mc = &portal->mc;
+
+	DPAA_ASSERT(mc->state == qman_mc_idle);
+#ifdef CONFIG_FSL_DPAA_CHECKING
+	mc->state = qman_mc_user;
+#endif
+	dpaa_zero(mc->cr);
+	return mc->cr;
+}
+
+static inline void qm_mc_commit(struct qm_portal *portal, u8 myverb)
+{
+	struct qm_mc *mc = &portal->mc;
+	union qm_mc_result *rr = mc->rr + mc->rridx;
+
+	DPAA_ASSERT(mc->state == qman_mc_user);
+	dma_wmb();
+	mc->cr->_ncw_verb = myverb | mc->vbit;
+	dpaa_flush(mc->cr);
+	dpaa_invalidate_touch_ro(rr);
+#ifdef CONFIG_FSL_DPAA_CHECKING
+	mc->state = qman_mc_hw;
+#endif
+}
+
+static inline union qm_mc_result *qm_mc_result(struct qm_portal *portal)
+{
+	struct qm_mc *mc = &portal->mc;
+	union qm_mc_result *rr = mc->rr + mc->rridx;
+
+	DPAA_ASSERT(mc->state == qman_mc_hw);
+	/*
+	 *  The inactive response register's verb byte always returns zero until
+	 * its command is submitted and completed. This includes the valid-bit,
+	 * in case you were wondering...
+	 */
+	if (!__raw_readb(&rr->verb)) {
+		dpaa_invalidate_touch_ro(rr);
+		return NULL;
+	}
+	mc->rridx ^= 1;
+	mc->vbit ^= QM_MCC_VERB_VBIT;
+#ifdef CONFIG_FSL_DPAA_CHECKING
+	mc->state = qman_mc_idle;
+#endif
+	return rr;
+}
+
+static inline int qm_mc_result_timeout(struct qm_portal *portal,
+				       union qm_mc_result **mcr)
+{
+	int timeout = QM_MCR_TIMEOUT;
+
+	do {
+		*mcr = qm_mc_result(portal);
+		if (*mcr)
+			break;
+		udelay(1);
+	} while (--timeout);
+
+	return timeout;
+}
+
+static inline void fq_set(struct qman_fq *fq, u32 mask)
+{
+	set_bits(mask, &fq->flags);
+}
+
+static inline void fq_clear(struct qman_fq *fq, u32 mask)
 {
+	clear_bits(mask, &fq->flags);
+}
+
+static inline int fq_isset(struct qman_fq *fq, u32 mask)
+{
+	return fq->flags & mask;
+}
+
+static inline int fq_isclear(struct qman_fq *fq, u32 mask)
+{
+	return !(fq->flags & mask);
+}
+
+struct qman_portal {
+	struct qm_portal p;
+	/* PORTAL_BITS_*** - dynamic, strictly internal */
+	unsigned long bits;
+	/* interrupt sources processed by portal_isr(), configurable */
+	unsigned long irq_sources;
+	u32 use_eqcr_ci_stashing;
+	/* only 1 volatile dequeue at a time */
+	struct qman_fq *vdqcr_owned;
+	u32 sdqcr;
+	/* probing time config params for cpu-affine portals */
+	const struct qm_portal_config *config;
+	/* 2-element array. cgrs[0] is mask, cgrs[1] is snapshot. */
+	struct qman_cgrs *cgrs;
+	/* linked-list of CSCN handlers. */
+	struct list_head cgr_cbs;
+	/* list lock */
+	spinlock_t cgr_lock;
+	struct work_struct congestion_work;
+	struct work_struct mr_work;
+	char irqname[MAX_IRQNAME];
+};
+
 #ifndef __rtems__
-	struct resource res;
+static cpumask_t affine_mask;
+static DEFINE_SPINLOCK(affine_mask_lock);
+static u16 affine_channels[NR_CPUS];
 #endif /* __rtems__ */
-	u32 __iomem *regs;
+static DEFINE_PER_CPU(struct qman_portal, qman_affine_portal);
+struct qman_portal *affine_portals[NR_CPUS];
+
+static inline struct qman_portal *get_affine_portal(void)
+{
+	return &get_cpu_var(qman_affine_portal);
+}
+
+static inline void put_affine_portal(void)
+{
+	put_cpu_var(qman_affine_portal);
+}
+
+static struct workqueue_struct *qm_portal_wq;
+
+int qman_wq_alloc(void)
+{
+	qm_portal_wq = alloc_workqueue("qman_portal_wq", 0, 1);
+	if (!qm_portal_wq)
+		return -ENOMEM;
+	return 0;
+}
+
+/*
+ * This is what everything can wait on, even if it migrates to a different cpu
+ * to the one whose affine portal it is waiting on.
+ */
+static DECLARE_WAIT_QUEUE_HEAD(affine_queue);
+
+static struct qman_fq **fq_table;
+static u32 num_fqids;
+
+int qman_alloc_fq_table(u32 _num_fqids)
+{
+	num_fqids = _num_fqids;
+
+	fq_table = vzalloc(num_fqids * 2 * sizeof(struct qman_fq *));
+	if (!fq_table)
+		return -ENOMEM;
+
+	pr_debug("Allocated fq lookup table at %p, entry count %u\n",
+		 fq_table, num_fqids * 2);
+	return 0;
+}
+
+static struct qman_fq *idx_to_fq(u32 idx)
+{
+	struct qman_fq *fq;
+
+#ifdef CONFIG_FSL_DPAA_CHECKING
+	if (WARN_ON(idx >= num_fqids * 2))
+		return NULL;
+#endif
+	fq = fq_table[idx];
+	DPAA_ASSERT(!fq || idx == fq->idx);
+
+	return fq;
+}
+
+/*
+ * Only returns full-service fq objects, not enqueue-only
+ * references (QMAN_FQ_FLAG_NO_MODIFY).
+ */
+static struct qman_fq *fqid_to_fq(u32 fqid)
+{
+	return idx_to_fq(fqid * 2);
+}
+
+static struct qman_fq *tag_to_fq(u32 tag)
+{
+#if BITS_PER_LONG == 64
+	return idx_to_fq(tag);
+#else
+	return (struct qman_fq *)tag;
+#endif
+}
+
+static u32 fq_to_tag(struct qman_fq *fq)
+{
+#if BITS_PER_LONG == 64
+	return fq->idx;
+#else
+	return (u32)fq;
+#endif
+}
+
+static u32 __poll_portal_slow(struct qman_portal *p, u32 is);
+static inline unsigned int __poll_portal_fast(struct qman_portal *p,
+					unsigned int poll_limit);
+static void qm_congestion_task(struct work_struct *work);
+static void qm_mr_process_task(struct work_struct *work);
+
+static irqreturn_t portal_isr(int irq, void *ptr)
+{
+	struct qman_portal *p = ptr;
+
+	u32 clear = QM_DQAVAIL_MASK | p->irq_sources;
+	u32 is = qm_in(&p->p, QM_REG_ISR) & p->irq_sources;
+
+	if (unlikely(!is))
+		return IRQ_NONE;
+
+	/* DQRR-handling if it's interrupt-driven */
+	if (is & QM_PIRQ_DQRI)
+		__poll_portal_fast(p, QMAN_POLL_LIMIT);
+	/* Handling of anything else that's interrupt-driven */
+	clear |= __poll_portal_slow(p, is);
+	qm_out(&p->p, QM_REG_ISR, clear);
+	return IRQ_HANDLED;
+}
+
+static int drain_mr_fqrni(struct qm_portal *p)
+{
+	const union qm_mr_entry *msg;
+loop:
+	msg = qm_mr_current(p);
+	if (!msg) {
+		/*
+		 * if MR was full and h/w had other FQRNI entries to produce, we
+		 * need to allow it time to produce those entries once the
+		 * existing entries are consumed. A worst-case situation
+		 * (fully-loaded system) means h/w sequencers may have to do 3-4
+		 * other things before servicing the portal's MR pump, each of
+		 * which (if slow) may take ~50 qman cycles (which is ~200
+		 * processor cycles). So rounding up and then multiplying this
+		 * worst-case estimate by a factor of 10, just to be
+		 * ultra-paranoid, goes as high as 10,000 cycles. NB, we consume
+		 * one entry at a time, so h/w has an opportunity to produce new
+		 * entries well before the ring has been fully consumed, so
+		 * we're being *really* paranoid here.
+		 */
+		u64 now, then = jiffies;
+
+		do {
+			now = jiffies;
+		} while ((then + 10000) > now);
+		msg = qm_mr_current(p);
+		if (!msg)
+			return 0;
+	}
+	if ((msg->verb & QM_MR_VERB_TYPE_MASK) != QM_MR_VERB_FQRNI) {
+		/* We aren't draining anything but FQRNIs */
+		pr_err("Found verb 0x%x in MR\n", msg->verb);
+		return -1;
+	}
+	qm_mr_next(p);
+	qm_mr_cci_consume(p, 1);
+	goto loop;
+}
+
+static int qman_create_portal(struct qman_portal *portal,
+			      const struct qm_portal_config *c,
+			      const struct qman_cgrs *cgrs)
+{
+	struct qm_portal *p;
+	int ret;
+	u32 isdr;
+
+	p = &portal->p;
+
+#ifdef CONFIG_FSL_PAMU
+	/* PAMU is required for stashing */
+	portal->use_eqcr_ci_stashing = ((qman_ip_rev >= QMAN_REV30) ? 1 : 0);
+#else
+	portal->use_eqcr_ci_stashing = 0;
+#endif
+	/*
+	 * prep the low-level portal struct with the mapped addresses from the
+	 * config, everything that follows depends on it and "config" is more
+	 * for (de)reference
+	 */
+	p->addr.ce = c->addr_virt[DPAA_PORTAL_CE];
+	p->addr.ci = c->addr_virt[DPAA_PORTAL_CI];
+	/*
+	 * If CI-stashing is used, the current defaults use a threshold of 3,
+	 * and stash with high-than-DQRR priority.
+	 */
+	if (qm_eqcr_init(p, qm_eqcr_pvb,
+			portal->use_eqcr_ci_stashing ? 3 : 0, 1)) {
+		dev_err(c->dev, "EQCR initialisation failed\n");
+		goto fail_eqcr;
+	}
+	if (qm_dqrr_init(p, c, qm_dqrr_dpush, qm_dqrr_pvb,
+			qm_dqrr_cdc, DQRR_MAXFILL)) {
+		dev_err(c->dev, "DQRR initialisation failed\n");
+		goto fail_dqrr;
+	}
+	if (qm_mr_init(p, qm_mr_pvb, qm_mr_cci)) {
+		dev_err(c->dev, "MR initialisation failed\n");
+		goto fail_mr;
+	}
+	if (qm_mc_init(p)) {
+		dev_err(c->dev, "MC initialisation failed\n");
+		goto fail_mc;
+	}
+	/* static interrupt-gating controls */
+	qm_dqrr_set_ithresh(p, QMAN_PIRQ_DQRR_ITHRESH);
+	qm_mr_set_ithresh(p, QMAN_PIRQ_MR_ITHRESH);
+	qm_out(p, QM_REG_ITPR, QMAN_PIRQ_IPERIOD);
+	portal->cgrs = kmalloc(2 * sizeof(*cgrs), GFP_KERNEL);
+	if (!portal->cgrs)
+		goto fail_cgrs;
+	/* initial snapshot is no-depletion */
+	qman_cgrs_init(&portal->cgrs[1]);
+	if (cgrs)
+		portal->cgrs[0] = *cgrs;
+	else
+		/* if the given mask is NULL, assume all CGRs can be seen */
+		qman_cgrs_fill(&portal->cgrs[0]);
+	INIT_LIST_HEAD(&portal->cgr_cbs);
+	spin_lock_init(&portal->cgr_lock);
+	INIT_WORK(&portal->congestion_work, qm_congestion_task);
+	INIT_WORK(&portal->mr_work, qm_mr_process_task);
+	portal->bits = 0;
+	portal->sdqcr = QM_SDQCR_SOURCE_CHANNELS | QM_SDQCR_COUNT_UPTO3 |
+			QM_SDQCR_DEDICATED_PRECEDENCE | QM_SDQCR_TYPE_PRIO_QOS |
+			QM_SDQCR_TOKEN_SET(0xab) | QM_SDQCR_CHANNELS_DEDICATED;
+	isdr = 0xffffffff;
+	qm_out(p, QM_REG_ISDR, isdr);
+	portal->irq_sources = 0;
+	qm_out(p, QM_REG_IER, 0);
+	qm_out(p, QM_REG_ISR, 0xffffffff);
+	snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu);
+	if (request_irq(c->irq, portal_isr, 0, portal->irqname,	portal)) {
+		dev_err(c->dev, "request_irq() failed\n");
+		goto fail_irq;
+	}
 #ifndef __rtems__
-	const char *s;
-	int ret, standby = 0;
+	if (c->cpu != -1 && irq_can_set_affinity(c->irq) &&
+	    irq_set_affinity(c->irq, cpumask_of(c->cpu))) {
+		dev_err(c->dev, "irq_set_affinity() failed\n");
+		goto fail_affinity;
+	}
 #endif /* __rtems__ */
-	u16 id;
-	u8 major, minor;
 
-#ifndef __rtems__
-	ret = of_address_to_resource(node, 0, &res);
+	/* Need EQCR to be empty before continuing */
+	isdr &= ~QM_PIRQ_EQCI;
+	qm_out(p, QM_REG_ISDR, isdr);
+	ret = qm_eqcr_get_fill(p);
 	if (ret) {
-		pr_err("Can't get %s property 'reg'\n", node->full_name);
-		return ret;
+		dev_err(c->dev, "EQCR unclean\n");
+		goto fail_eqcr_empty;
 	}
-	s = of_get_property(node, "fsl,hv-claimable", &ret);
-	if (s && !strcmp(s, "standby"))
-		standby = 1;
-	if (!standby) {
-		ret = parse_mem_property(node, &fqd_a, &fqd_sz, 1);
-		BUG_ON(ret);
-		ret = parse_mem_property(node, &pfdr_a, &pfdr_sz, 0);
-		BUG_ON(ret);
+	isdr &= ~(QM_PIRQ_DQRI | QM_PIRQ_MRI);
+	qm_out(p, QM_REG_ISDR, isdr);
+	if (qm_dqrr_current(p)) {
+		dev_err(c->dev, "DQRR unclean\n");
+		qm_dqrr_cdc_consume_n(p, 0xffff);
 	}
-#else /* __rtems__ */
-	memset((void *)fqd_a, 0, fqd_sz);
+	if (qm_mr_current(p) && drain_mr_fqrni(p)) {
+		/* special handling, drain just in case it's a few FQRNIs */
+		const union qm_mr_entry *e = qm_mr_current(p);
+
+		dev_err(c->dev, "MR dirty, VB 0x%x, rc 0x%x, addr 0x%llx\n",
+			e->verb, e->ern.rc, qm_fd_addr_get64(&e->ern.fd));
+		goto fail_dqrr_mr_empty;
+	}
+	/* Success */
+	portal->config = c;
+	qm_out(p, QM_REG_ISDR, 0);
+	qm_out(p, QM_REG_IIR, 0);
+	/* Write a sane SDQCR */
+	qm_dqrr_sdqcr_set(p, portal->sdqcr);
+	return 0;
+
+fail_dqrr_mr_empty:
+fail_eqcr_empty:
+#ifndef __rtems__
+fail_affinity:
 #endif /* __rtems__ */
-	/* Global configuration */
+	free_irq(c->irq, portal);
+fail_irq:
+	kfree(portal->cgrs);
+fail_cgrs:
+	qm_mc_finish(p);
+fail_mc:
+	qm_mr_finish(p);
+fail_mr:
+	qm_dqrr_finish(p);
+fail_dqrr:
+	qm_eqcr_finish(p);
+fail_eqcr:
+	return -EIO;
+}
+
+struct qman_portal *qman_create_affine_portal(const struct qm_portal_config *c,
+					      const struct qman_cgrs *cgrs)
+{
+	struct qman_portal *portal;
+	int err;
+
+	portal = &per_cpu(qman_affine_portal, c->cpu);
+	err = qman_create_portal(portal, c, cgrs);
+	if (err)
+		return NULL;
+
 #ifndef __rtems__
-	regs = ioremap(res.start, res.end - res.start + 1);
-#else /* __rtems__ */
-	regs = (u32 __iomem *)&qoriq.qman;
+	spin_lock(&affine_mask_lock);
+	cpumask_set_cpu(c->cpu, &affine_mask);
+	affine_channels[c->cpu] = c->channel;
 #endif /* __rtems__ */
-	qm = qm_create(regs);
+	affine_portals[c->cpu] = portal;
 #ifndef __rtems__
-	qm_node = node;
+	spin_unlock(&affine_mask_lock);
 #endif /* __rtems__ */
-	qm_get_version(qm, &id, &major, &minor);
-	pr_info("Ver: %04x,%02x,%02x\n", id, major, minor);
-	if (!qman_ip_rev) {
-		if ((major == 1) && (minor == 0)) {
-			pr_err("Rev1.0 on P4080 rev1 is not supported!\n");
+
+	return portal;
+}
+
+static void qman_destroy_portal(struct qman_portal *qm)
+{
+	const struct qm_portal_config *pcfg;
+
+	/* Stop dequeues on the portal */
+	qm_dqrr_sdqcr_set(&qm->p, 0);
+
+	/*
+	 * NB we do this to "quiesce" EQCR. If we add enqueue-completions or
+	 * something related to QM_PIRQ_EQCI, this may need fixing.
+	 * Also, due to the prefetching model used for CI updates in the enqueue
+	 * path, this update will only invalidate the CI cacheline *after*
+	 * working on it, so we need to call this twice to ensure a full update
+	 * irrespective of where the enqueue processing was at when the teardown
+	 * began.
+	 */
+	qm_eqcr_cce_update(&qm->p);
+	qm_eqcr_cce_update(&qm->p);
+	pcfg = qm->config;
+
+	free_irq(pcfg->irq, qm);
+
+	kfree(qm->cgrs);
+	qm_mc_finish(&qm->p);
+	qm_mr_finish(&qm->p);
+	qm_dqrr_finish(&qm->p);
+	qm_eqcr_finish(&qm->p);
+
+	qm->config = NULL;
+}
+
+const struct qm_portal_config *qman_destroy_affine_portal(void)
+{
+	struct qman_portal *qm = get_affine_portal();
+	const struct qm_portal_config *pcfg;
+	int cpu;
+
+	pcfg = qm->config;
+	cpu = pcfg->cpu;
+
+	qman_destroy_portal(qm);
+
 #ifndef __rtems__
-			iounmap(regs);
+	spin_lock(&affine_mask_lock);
+	cpumask_clear_cpu(cpu, &affine_mask);
+	spin_unlock(&affine_mask_lock);
+#else /* __rtems__ */
+	(void)cpu;
 #endif /* __rtems__ */
-			return -ENODEV;
-		} else if ((major == 1) && (minor == 1))
-			qman_ip_rev = QMAN_REV11;
-		else if	((major == 1) && (minor == 2))
-			qman_ip_rev = QMAN_REV12;
-		else if ((major == 2) && (minor == 0))
-			qman_ip_rev = QMAN_REV20;
-		else if ((major == 3) && (minor == 0))
-			qman_ip_rev = QMAN_REV30;
-		else if ((major == 3) && (minor == 1))
-			qman_ip_rev = QMAN_REV31;
-		else {
-			pr_warn("Unknown version, default to rev1.1\n");
-			qman_ip_rev = QMAN_REV11;
-		}
+	put_affine_portal();
+	return pcfg;
+}
+
+/* Inline helper to reduce nesting in __poll_portal_slow() */
+static inline void fq_state_change(struct qman_portal *p, struct qman_fq *fq,
+				   const union qm_mr_entry *msg, u8 verb)
+{
+	switch (verb) {
+	case QM_MR_VERB_FQRL:
+		DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_ORL));
+		fq_clear(fq, QMAN_FQ_STATE_ORL);
+		break;
+	case QM_MR_VERB_FQRN:
+		DPAA_ASSERT(fq->state == qman_fq_state_parked ||
+			    fq->state == qman_fq_state_sched);
+		DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_CHANGING));
+		fq_clear(fq, QMAN_FQ_STATE_CHANGING);
+		if (msg->fq.fqs & QM_MR_FQS_NOTEMPTY)
+			fq_set(fq, QMAN_FQ_STATE_NE);
+		if (msg->fq.fqs & QM_MR_FQS_ORLPRESENT)
+			fq_set(fq, QMAN_FQ_STATE_ORL);
+		fq->state = qman_fq_state_retired;
+		break;
+	case QM_MR_VERB_FQPN:
+		DPAA_ASSERT(fq->state == qman_fq_state_sched);
+		DPAA_ASSERT(fq_isclear(fq, QMAN_FQ_STATE_CHANGING));
+		fq->state = qman_fq_state_parked;
 	}
+}
 
-#ifndef __rtems__
-	if (standby) {
-		pr_info("  -> in standby mode\n");
-		return 0;
+static void qm_congestion_task(struct work_struct *work)
+{
+	struct qman_portal *p = container_of(work, struct qman_portal,
+					     congestion_work);
+	struct qman_cgrs rr, c;
+	union qm_mc_result *mcr;
+	struct qman_cgr *cgr;
+
+	spin_lock(&p->cgr_lock);
+	qm_mc_start(&p->p);
+	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION);
+	if (!qm_mc_result_timeout(&p->p, &mcr)) {
+		spin_unlock(&p->cgr_lock);
+		dev_crit(p->config->dev, "QUERYCONGESTION timeout\n");
+		return;
 	}
-#endif /* __rtems__ */
-	return 0;
+	/* mask out the ones I'm not interested in */
+	qman_cgrs_and(&rr, (struct qman_cgrs *)&mcr->querycongestion.state,
+		      &p->cgrs[0]);
+	/* check previous snapshot for delta, enter/exit congestion */
+	qman_cgrs_xor(&c, &rr, &p->cgrs[1]);
+	/* update snapshot */
+	qman_cgrs_cp(&p->cgrs[1], &rr);
+	/* Invoke callback */
+	list_for_each_entry(cgr, &p->cgr_cbs, node)
+		if (cgr->cb && qman_cgrs_get(&c, cgr->cgrid))
+			cgr->cb(p, cgr, qman_cgrs_get(&rr, cgr->cgrid));
+	spin_unlock(&p->cgr_lock);
 }
 
-int qman_have_ccsr(void)
+static void qm_mr_process_task(struct work_struct *work)
 {
-	return qm ? 1 : 0;
+	struct qman_portal *p = container_of(work, struct qman_portal,
+					     mr_work);
+	const union qm_mr_entry *msg;
+	struct qman_fq *fq;
+	u8 verb, num = 0;
+
+	preempt_disable();
+
+	while (1) {
+		qm_mr_pvb_update(&p->p);
+		msg = qm_mr_current(&p->p);
+		if (!msg)
+			break;
+
+		verb = msg->verb & QM_MR_VERB_TYPE_MASK;
+		/* The message is a software ERN iff the 0x20 bit is clear */
+		if (verb & 0x20) {
+			switch (verb) {
+			case QM_MR_VERB_FQRNI:
+				/* nada, we drop FQRNIs on the floor */
+				break;
+			case QM_MR_VERB_FQRN:
+			case QM_MR_VERB_FQRL:
+				/* Lookup in the retirement table */
+				fq = fqid_to_fq(qm_fqid_get(&msg->fq));
+				if (WARN_ON(!fq))
+					break;
+				fq_state_change(p, fq, msg, verb);
+				if (fq->cb.fqs)
+					fq->cb.fqs(p, fq, msg);
+				break;
+			case QM_MR_VERB_FQPN:
+				/* Parked */
+				fq = tag_to_fq(be32_to_cpu(msg->fq.context_b));
+				fq_state_change(p, fq, msg, verb);
+				if (fq->cb.fqs)
+					fq->cb.fqs(p, fq, msg);
+				break;
+			case QM_MR_VERB_DC_ERN:
+				/* DCP ERN */
+				pr_crit_once("Leaking DCP ERNs!\n");
+				break;
+			default:
+				pr_crit("Invalid MR verb 0x%02x\n", verb);
+			}
+		} else {
+			/* Its a software ERN */
+			fq = tag_to_fq(be32_to_cpu(msg->ern.tag));
+			fq->cb.ern(p, fq, msg);
+		}
+		num++;
+		qm_mr_next(&p->p);
+	}
+
+	qm_mr_cci_consume(&p->p, num);
+	preempt_enable();
 }
 
-#ifndef __rtems__
-__init void qman_init_early(void)
+static u32 __poll_portal_slow(struct qman_portal *p, u32 is)
 {
-	struct device_node *dn;
-	int ret;
+	if (is & QM_PIRQ_CSCI) {
+		queue_work_on(smp_processor_id(), qm_portal_wq,
+			      &p->congestion_work);
+	}
 
-	for_each_compatible_node(dn, NULL, "fsl,qman") {
-		if (qm)
-			pr_err("%s: only one 'fsl,qman' allowed\n",
-				dn->full_name);
-		else {
-			if (!of_device_is_available(dn))
-				continue;
+	if (is & QM_PIRQ_EQRI) {
+		qm_eqcr_cce_update(&p->p);
+		qm_eqcr_set_ithresh(&p->p, 0);
+		wake_up(&affine_queue);
+	}
 
-			ret = fsl_qman_init(dn);
-			BUG_ON(ret);
-		}
+	if (is & QM_PIRQ_MRI) {
+		queue_work_on(smp_processor_id(), qm_portal_wq,
+			      &p->mr_work);
 	}
+
+	return is;
 }
-#else /* __rtems__ */
-#include <bsp/fdt.h>
 
-static void
-qman_sysinit(void)
+/*
+ * remove some slowish-path stuff from the "fast path" and make sure it isn't
+ * inlined.
+ */
+static noinline void clear_vdqcr(struct qman_portal *p, struct qman_fq *fq)
 {
-	const char *fdt = bsp_fdt_get();
-	struct device_node dn;
-	const char *name;
-	int node;
-	int ret;
+	p->vdqcr_owned = NULL;
+	fq_clear(fq, QMAN_FQ_STATE_VDQCR);
+	wake_up(&affine_queue);
+}
+
+/*
+ * The only states that would conflict with other things if they ran at the
+ * same time on the same cpu are:
+ *
+ *   (i) setting/clearing vdqcr_owned, and
+ *  (ii) clearing the NE (Not Empty) flag.
+ *
+ * Both are safe. Because;
+ *
+ *   (i) this clearing can only occur after qman_volatile_dequeue() has set the
+ *	 vdqcr_owned field (which it does before setting VDQCR), and
+ *	 qman_volatile_dequeue() blocks interrupts and preemption while this is
+ *	 done so that we can't interfere.
+ *  (ii) the NE flag is only cleared after qman_retire_fq() has set it, and as
+ *	 with (i) that API prevents us from interfering until it's safe.
+ *
+ * The good thing is that qman_volatile_dequeue() and qman_retire_fq() run far
+ * less frequently (ie. per-FQ) than __poll_portal_fast() does, so the nett
+ * advantage comes from this function not having to "lock" anything at all.
+ *
+ * Note also that the callbacks are invoked at points which are safe against the
+ * above potential conflicts, but that this function itself is not re-entrant
+ * (this is because the function tracks one end of each FIFO in the portal and
+ * we do *not* want to lock that). So the consequence is that it is safe for
+ * user callbacks to call into any QMan API.
+ */
+static inline unsigned int __poll_portal_fast(struct qman_portal *p,
+					unsigned int poll_limit)
+{
+	const struct qm_dqrr_entry *dq;
+	struct qman_fq *fq;
+	enum qman_cb_dqrr_result res;
+	unsigned int limit = 0;
 
-	memset(&dn, 0, sizeof(dn));
+	do {
+		qm_dqrr_pvb_update(&p->p);
+		dq = qm_dqrr_current(&p->p);
+		if (!dq)
+			break;
+
+		if (dq->stat & QM_DQRR_STAT_UNSCHEDULED) {
+			/*
+			 * VDQCR: don't trust context_b as the FQ may have
+			 * been configured for h/w consumption and we're
+			 * draining it post-retirement.
+			 */
+			fq = p->vdqcr_owned;
+			/*
+			 * We only set QMAN_FQ_STATE_NE when retiring, so we
+			 * only need to check for clearing it when doing
+			 * volatile dequeues.  It's one less thing to check
+			 * in the critical path (SDQCR).
+			 */
+			if (dq->stat & QM_DQRR_STAT_FQ_EMPTY)
+				fq_clear(fq, QMAN_FQ_STATE_NE);
+			/*
+			 * This is duplicated from the SDQCR code, but we
+			 * have stuff to do before *and* after this callback,
+			 * and we don't want multiple if()s in the critical
+			 * path (SDQCR).
+			 */
+			res = fq->cb.dqrr(p, fq, dq);
+			if (res == qman_cb_dqrr_stop)
+				break;
+			/* Check for VDQCR completion */
+			if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED)
+				clear_vdqcr(p, fq);
+		} else {
+			/* SDQCR: context_b points to the FQ */
+			fq = tag_to_fq(be32_to_cpu(dq->context_b));
+			/* Now let the callback do its stuff */
+			res = fq->cb.dqrr(p, fq, dq);
+			/*
+			 * The callback can request that we exit without
+			 * consuming this entry nor advancing;
+			 */
+			if (res == qman_cb_dqrr_stop)
+				break;
+		}
+		/* Interpret 'dq' from a driver perspective. */
+		/*
+		 * Parking isn't possible unless HELDACTIVE was set. NB,
+		 * FORCEELIGIBLE implies HELDACTIVE, so we only need to
+		 * check for HELDACTIVE to cover both.
+		 */
+		DPAA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) ||
+			    (res != qman_cb_dqrr_park));
+		/* just means "skip it, I'll consume it myself later on" */
+		if (res != qman_cb_dqrr_defer)
+			qm_dqrr_cdc_consume_1ptr(&p->p, dq,
+						 res == qman_cb_dqrr_park);
+		/* Move forward */
+		qm_dqrr_next(&p->p);
+		/*
+		 * Entry processed and consumed, increment our counter.  The
+		 * callback can request that we exit after consuming the
+		 * entry, and we also exit if we reach our processing limit,
+		 * so loop back only if neither of these conditions is met.
+		 */
+	} while (++limit < poll_limit && res != qman_cb_dqrr_consume_stop);
+
+	return limit;
+}
 
-	name = "fsl,qman";
-	node = fdt_node_offset_by_compatible(fdt, 0, name);
-	if (node < 0)
-		panic("qman: no qman in FDT");
+void qman_p_irqsource_add(struct qman_portal *p, u32 bits)
+{
+	unsigned long irqflags;
 
-	dn.full_name = name;
-	dn.offset = node;
+	local_irq_save(irqflags);
+	set_bits(bits & QM_PIRQ_VISIBLE, &p->irq_sources);
+	qm_out(&p->p, QM_REG_IER, p->irq_sources);
+	local_irq_restore(irqflags);
+}
+EXPORT_SYMBOL(qman_p_irqsource_add);
 
-	ret = fsl_qman_init(&dn);
-	if (ret != 0)
-		panic("qman: init 1 failed");
+void qman_p_irqsource_remove(struct qman_portal *p, u32 bits)
+{
+	unsigned long irqflags;
+	u32 ier;
+
+	/*
+	 * Our interrupt handler only processes+clears status register bits that
+	 * are in p->irq_sources. As we're trimming that mask, if one of them
+	 * were to assert in the status register just before we remove it from
+	 * the enable register, there would be an interrupt-storm when we
+	 * release the IRQ lock. So we wait for the enable register update to
+	 * take effect in h/w (by reading it back) and then clear all other bits
+	 * in the status register. Ie. we clear them from ISR once it's certain
+	 * IER won't allow them to reassert.
+	 */
+	local_irq_save(irqflags);
+	bits &= QM_PIRQ_VISIBLE;
+	clear_bits(bits, &p->irq_sources);
+	qm_out(&p->p, QM_REG_IER, p->irq_sources);
+	ier = qm_in(&p->p, QM_REG_IER);
+	/*
+	 * Using "~ier" (rather than "bits" or "~p->irq_sources") creates a
+	 * data-dependency, ie. to protect against re-ordering.
+	 */
+	qm_out(&p->p, QM_REG_ISR, ~ier);
+	local_irq_restore(irqflags);
+}
+EXPORT_SYMBOL(qman_p_irqsource_remove);
 
-	ret = qman_init_ccsr(&dn);
-	if (ret != 0)
-		panic("qman: init CCSR failed");
+#ifndef __rtems__
+const cpumask_t *qman_affine_cpus(void)
+{
+	return &affine_mask;
+}
+EXPORT_SYMBOL(qman_affine_cpus);
 
-	ret = qman_init(&dn);
-	if (ret != 0)
-		panic("qman: init 2 failed");
+u16 qman_affine_channel(int cpu)
+{
+	if (cpu < 0) {
+		struct qman_portal *portal = get_affine_portal();
 
-	ret = qman_resource_init();
-	if (ret != 0)
-		panic("qman: resource init failed");
+		cpu = portal->config->cpu;
+		put_affine_portal();
+	}
+	WARN_ON(!cpumask_test_cpu(cpu, &affine_mask));
+	return affine_channels[cpu];
 }
-SYSINIT(qman_sysinit, SI_SUB_CPU, SI_ORDER_SECOND, qman_sysinit, NULL);
+EXPORT_SYMBOL(qman_affine_channel);
 #endif /* __rtems__ */
 
-static void log_edata_bits(u32 bit_count)
+struct qman_portal *qman_get_affine_portal(int cpu)
+{
+	return affine_portals[cpu];
+}
+EXPORT_SYMBOL(qman_get_affine_portal);
+
+int qman_p_poll_dqrr(struct qman_portal *p, unsigned int limit)
 {
-	u32 i, j, mask = 0xffffffff;
+	return __poll_portal_fast(p, limit);
+}
+EXPORT_SYMBOL(qman_p_poll_dqrr);
 
-	pr_warn("ErrInt, EDATA:\n");
-	i = bit_count/32;
-	if (bit_count%32) {
-		i++;
-		mask = ~(mask << bit_count%32);
+void qman_p_static_dequeue_add(struct qman_portal *p, u32 pools)
+{
+	unsigned long irqflags;
+
+	local_irq_save(irqflags);
+	pools &= p->config->pools;
+	p->sdqcr |= pools;
+	qm_dqrr_sdqcr_set(&p->p, p->sdqcr);
+	local_irq_restore(irqflags);
+}
+EXPORT_SYMBOL(qman_p_static_dequeue_add);
+
+/* Frame queue API */
+
+static const char *mcr_result_str(u8 result)
+{
+	switch (result) {
+	case QM_MCR_RESULT_NULL:
+		return "QM_MCR_RESULT_NULL";
+	case QM_MCR_RESULT_OK:
+		return "QM_MCR_RESULT_OK";
+	case QM_MCR_RESULT_ERR_FQID:
+		return "QM_MCR_RESULT_ERR_FQID";
+	case QM_MCR_RESULT_ERR_FQSTATE:
+		return "QM_MCR_RESULT_ERR_FQSTATE";
+	case QM_MCR_RESULT_ERR_NOTEMPTY:
+		return "QM_MCR_RESULT_ERR_NOTEMPTY";
+	case QM_MCR_RESULT_PENDING:
+		return "QM_MCR_RESULT_PENDING";
+	case QM_MCR_RESULT_ERR_BADCOMMAND:
+		return "QM_MCR_RESULT_ERR_BADCOMMAND";
 	}
-	j = 16-i;
-	pr_warn("  0x%08x\n", qm_in(EDATA(j)) & mask);
-	j++;
-	for (; j < 16; j++)
-		pr_warn("  0x%08x\n", qm_in(EDATA(j)));
+	return "<unknown MCR result>";
 }
 
-static void log_additional_error_info(u32 isr_val, u32 ecsr_val)
+int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq)
 {
-	union qman_ecir ecir_val;
-	union qman_eadr eadr_val;
+	if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) {
+		int ret = qman_alloc_fqid(&fqid);
+
+		if (ret)
+			return ret;
+	}
+	fq->fqid = fqid;
+	fq->flags = flags;
+	fq->state = qman_fq_state_oos;
+	fq->cgr_groupid = 0;
+
+	/* A context_b of 0 is allegedly special, so don't use that fqid */
+	if (fqid == 0 || fqid >= num_fqids) {
+		WARN(1, "bad fqid %d\n", fqid);
+		return -EINVAL;
+	}
 
-	ecir_val.ecir_raw = qm_in(ECIR);
-	/* Is portal info valid */
-	if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) {
-		union qman_ecir2 ecir2_val;
+	fq->idx = fqid * 2;
+	if (flags & QMAN_FQ_FLAG_NO_MODIFY)
+		fq->idx++;
 
-		ecir2_val.ecir2_raw = qm_in(ECIR2);
-		if (ecsr_val & PORTAL_ECSR_ERR) {
-			pr_warn("ErrInt: %s id %d\n",
-				ecir2_val.info.portal_type ? "DCP" : "SWP",
-				ecir2_val.info.portal_num);
-		}
-		if (ecsr_val & (FQID_ECSR_ERR | QM_EIRQ_IECE))
-			pr_warn("ErrInt: ecir.fqid 0x%x\n", ecir_val.info.fqid);
-
-		if (ecsr_val & (QM_EIRQ_SBEI|QM_EIRQ_MBEI)) {
-			eadr_val.eadr_raw = qm_in(EADR);
-			pr_warn("ErrInt: EADR Memory: %s, 0x%x\n",
-				error_mdata[eadr_val.info_rev3.memid].txt,
-				error_mdata[eadr_val.info_rev3.memid].addr_mask
-					& eadr_val.info_rev3.eadr);
-			log_edata_bits(
-				error_mdata[eadr_val.info_rev3.memid].bits);
-		}
-	} else {
-		if (ecsr_val & PORTAL_ECSR_ERR) {
-			pr_warn("ErrInt: %s id %d\n",
-				ecir_val.info.portal_type ? "DCP" : "SWP",
-				ecir_val.info.portal_num);
-		}
-		if (ecsr_val & FQID_ECSR_ERR)
-			pr_warn("ErrInt: ecir.fqid 0x%x\n", ecir_val.info.fqid);
-
-		if (ecsr_val & (QM_EIRQ_SBEI|QM_EIRQ_MBEI)) {
-			eadr_val.eadr_raw = qm_in(EADR);
-			pr_warn("ErrInt: EADR Memory: %s, 0x%x\n",
-				error_mdata[eadr_val.info.memid].txt,
-				error_mdata[eadr_val.info.memid].addr_mask
-					& eadr_val.info.eadr);
-			log_edata_bits(error_mdata[eadr_val.info.memid].bits);
-		}
+	WARN_ON(fq_table[fq->idx]);
+	fq_table[fq->idx] = fq;
+
+	return 0;
+}
+EXPORT_SYMBOL(qman_create_fq);
+
+void qman_destroy_fq(struct qman_fq *fq)
+{
+	/*
+	 * We don't need to lock the FQ as it is a pre-condition that the FQ be
+	 * quiesced. Instead, run some checks.
+	 */
+	switch (fq->state) {
+	case qman_fq_state_parked:
+	case qman_fq_state_oos:
+		if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID))
+			qman_release_fqid(fq->fqid);
+
+		DPAA_ASSERT(fq_table[fq->idx]);
+		fq_table[fq->idx] = NULL;
+		return;
+	default:
+		break;
 	}
+	DPAA_ASSERT(NULL == "qman_free_fq() on unquiesced FQ!");
 }
+EXPORT_SYMBOL(qman_destroy_fq);
 
-/* QMan interrupt handler */
-static irqreturn_t qman_isr(int irq, void *ptr)
+u32 qman_fq_fqid(struct qman_fq *fq)
 {
-	u32 isr_val, ier_val, ecsr_val, isr_mask, i;
+	return fq->fqid;
+}
+EXPORT_SYMBOL(qman_fq_fqid);
 
-	ier_val = qm_err_isr_enable_read(qm);
-	isr_val = qm_err_isr_status_read(qm);
-	ecsr_val = qm_in(ECSR);
-	isr_mask = isr_val & ier_val;
+int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts)
+{
+	union qm_mc_command *mcc;
+	union qm_mc_result *mcr;
+	struct qman_portal *p;
+	u8 res, myverb;
+	int ret = 0;
 
-	if (!isr_mask)
-		return IRQ_NONE;
-	for (i = 0; i < QMAN_HWE_COUNT; i++) {
-		if (qman_hwerr_txts[i].mask & isr_mask) {
-			pr_warn("ErrInt: %s\n", qman_hwerr_txts[i].txt);
-			if (qman_hwerr_txts[i].mask & ecsr_val) {
-				log_additional_error_info(isr_mask, ecsr_val);
-				/* Re-arm error capture registers */
-				qm_out(ECSR, ecsr_val);
-			}
-			if (qman_hwerr_txts[i].mask & QMAN_ERRS_TO_UNENABLE) {
-				pr_devel("Un-enabling error 0x%x\n",
-					 qman_hwerr_txts[i].mask);
-				ier_val &= ~qman_hwerr_txts[i].mask;
-				qm_err_isr_enable_write(qm, ier_val);
+	myverb = (flags & QMAN_INITFQ_FLAG_SCHED)
+		? QM_MCC_VERB_INITFQ_SCHED : QM_MCC_VERB_INITFQ_PARKED;
+
+	if (fq->state != qman_fq_state_oos &&
+	    fq->state != qman_fq_state_parked)
+		return -EINVAL;
+#ifdef CONFIG_FSL_DPAA_CHECKING
+	if (fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))
+		return -EINVAL;
+#endif
+	if (opts && (be16_to_cpu(opts->we_mask) & QM_INITFQ_WE_OAC)) {
+		/* And can't be set at the same time as TDTHRESH */
+		if (be16_to_cpu(opts->we_mask) & QM_INITFQ_WE_TDTHRESH)
+			return -EINVAL;
+	}
+	/* Issue an INITFQ_[PARKED|SCHED] management command */
+	p = get_affine_portal();
+	if (fq_isset(fq, QMAN_FQ_STATE_CHANGING) ||
+	    (fq->state != qman_fq_state_oos &&
+	     fq->state != qman_fq_state_parked)) {
+		ret = -EBUSY;
+		goto out;
+	}
+	mcc = qm_mc_start(&p->p);
+	if (opts)
+		mcc->initfq = *opts;
+	qm_fqid_set(&mcc->fq, fq->fqid);
+	mcc->initfq.count = 0;
+	/*
+	 * If the FQ does *not* have the TO_DCPORTAL flag, context_b is set as a
+	 * demux pointer. Otherwise, the caller-provided value is allowed to
+	 * stand, don't overwrite it.
+	 */
+	if (fq_isclear(fq, QMAN_FQ_FLAG_TO_DCPORTAL)) {
+		dma_addr_t phys_fq;
+
+		mcc->initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_CONTEXTB);
+		mcc->initfq.fqd.context_b = cpu_to_be32(fq_to_tag(fq));
+		/*
+		 *  and the physical address - NB, if the user wasn't trying to
+		 * set CONTEXTA, clear the stashing settings.
+		 */
+		if (!(be16_to_cpu(mcc->initfq.we_mask) &
+				  QM_INITFQ_WE_CONTEXTA)) {
+			mcc->initfq.we_mask |=
+				cpu_to_be16(QM_INITFQ_WE_CONTEXTA);
+			memset(&mcc->initfq.fqd.context_a, 0,
+				sizeof(mcc->initfq.fqd.context_a));
+		} else {
+#ifndef __rtems__
+			struct qman_portal *p = qman_dma_portal;
+
+			phys_fq = dma_map_single(p->config->dev, fq,
+						 sizeof(*fq), DMA_TO_DEVICE);
+			if (dma_mapping_error(p->config->dev, phys_fq)) {
+				dev_err(p->config->dev, "dma_mapping failed\n");
+				ret = -EIO;
+				goto out;
 			}
+#else /* __rtems__ */
+			phys_fq = (dma_addr_t)fq;
+#endif /* __rtems__ */
+
+			qm_fqd_stashing_set64(&mcc->initfq.fqd, phys_fq);
 		}
 	}
-	qm_err_isr_status_clear(qm, isr_val);
-	return IRQ_HANDLED;
+	if (flags & QMAN_INITFQ_FLAG_LOCAL) {
+		int wq = 0;
+
+		if (!(be16_to_cpu(mcc->initfq.we_mask) &
+				  QM_INITFQ_WE_DESTWQ)) {
+			mcc->initfq.we_mask |=
+				cpu_to_be16(QM_INITFQ_WE_DESTWQ);
+			wq = 4;
+		}
+		qm_fqd_set_destwq(&mcc->initfq.fqd, p->config->channel, wq);
+	}
+	qm_mc_commit(&p->p, myverb);
+	if (!qm_mc_result_timeout(&p->p, &mcr)) {
+		dev_err(p->config->dev, "MCR timeout\n");
+		ret = -ETIMEDOUT;
+		goto out;
+	}
+
+	DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
+	res = mcr->result;
+	if (res != QM_MCR_RESULT_OK) {
+		ret = -EIO;
+		goto out;
+	}
+	if (opts) {
+		if (be16_to_cpu(opts->we_mask) & QM_INITFQ_WE_FQCTRL) {
+			if (be16_to_cpu(opts->fqd.fq_ctrl) & QM_FQCTRL_CGE)
+				fq_set(fq, QMAN_FQ_STATE_CGR_EN);
+			else
+				fq_clear(fq, QMAN_FQ_STATE_CGR_EN);
+		}
+		if (be16_to_cpu(opts->we_mask) & QM_INITFQ_WE_CGID)
+			fq->cgr_groupid = opts->fqd.cgid;
+	}
+	fq->state = (flags & QMAN_INITFQ_FLAG_SCHED) ?
+		qman_fq_state_sched : qman_fq_state_parked;
+
+out:
+	put_affine_portal();
+	return ret;
 }
+EXPORT_SYMBOL(qman_init_fq);
 
-static int __bind_irq(struct device_node *node)
+int qman_schedule_fq(struct qman_fq *fq)
 {
-	int ret, err_irq;
+	union qm_mc_command *mcc;
+	union qm_mc_result *mcr;
+	struct qman_portal *p;
+	int ret = 0;
 
-	err_irq = of_irq_to_resource(node, 0, NULL);
-	if (err_irq == NO_IRQ) {
-		pr_info("Can't get %s property 'interrupts'\n",
-			node->full_name);
-		return -ENODEV;
+	if (fq->state != qman_fq_state_parked)
+		return -EINVAL;
+#ifdef CONFIG_FSL_DPAA_CHECKING
+	if (fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))
+		return -EINVAL;
+#endif
+	/* Issue a ALTERFQ_SCHED management command */
+	p = get_affine_portal();
+	if (fq_isset(fq, QMAN_FQ_STATE_CHANGING) ||
+	    fq->state != qman_fq_state_parked) {
+		ret = -EBUSY;
+		goto out;
 	}
-	ret = request_irq(err_irq, qman_isr, IRQF_SHARED, "qman-err", node);
-	if (ret)  {
-		pr_err("request_irq() failed %d for '%s'\n",
-		       ret, node->full_name);
-		return -ENODEV;
+	mcc = qm_mc_start(&p->p);
+	qm_fqid_set(&mcc->fq, fq->fqid);
+	qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_SCHED);
+	if (!qm_mc_result_timeout(&p->p, &mcr)) {
+		dev_err(p->config->dev, "ALTER_SCHED timeout\n");
+		ret = -ETIMEDOUT;
+		goto out;
 	}
-	/* Write-to-clear any stale bits, (eg. starvation being asserted prior
-	 * to resource allocation during driver init). */
-	qm_err_isr_status_clear(qm, 0xffffffff);
-	/* Enable Error Interrupts */
-	qm_err_isr_enable_write(qm, 0xffffffff);
-	return 0;
+
+	DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_SCHED);
+	if (mcr->result != QM_MCR_RESULT_OK) {
+		ret = -EIO;
+		goto out;
+	}
+	fq->state = qman_fq_state_sched;
+out:
+	put_affine_portal();
+	return ret;
 }
+EXPORT_SYMBOL(qman_schedule_fq);
 
-int qman_init_ccsr(struct device_node *node)
+int qman_retire_fq(struct qman_fq *fq, u32 *flags)
 {
+	union qm_mc_command *mcc;
+	union qm_mc_result *mcr;
+	struct qman_portal *p;
 	int ret;
+	u8 res;
 
-#ifndef __rtems__
-	if (!qman_have_ccsr())
-		return 0;
-	if (node != qm_node)
+	if (fq->state != qman_fq_state_parked &&
+	    fq->state != qman_fq_state_sched)
 		return -EINVAL;
-#endif /* __rtems__ */
-	/* FQD memory */
-	qm_set_memory(qm, qm_memory_fqd, fqd_a, 1, 0, 0, fqd_sz);
-	/* PFDR memory */
-	qm_set_memory(qm, qm_memory_pfdr, pfdr_a, 1, 0, 0, pfdr_sz);
-	qm_init_pfdr(qm, 8, pfdr_sz / 64 - 8);
-	/* thresholds */
-	qm_set_pfdr_threshold(qm, 512, 64);
-	qm_set_sfdr_threshold(qm, 128);
-	/* clear stale PEBI bit from interrupt status register */
-	qm_err_isr_status_clear(qm, QM_EIRQ_PEBI);
-	/* corenet initiator settings */
-	qm_set_corenet_initiator(qm);
-	/* HID settings */
-	qm_set_hid(qm);
-	/* Set scheduling weights to defaults */
-	for (ret = qm_wq_first; ret <= qm_wq_last; ret++)
-		qm_set_wq_scheduling(qm, ret, 0, 0, 0, 0, 0, 0, 0);
-	/* We are not prepared to accept ERNs for hardware enqueues */
-	qm_set_dc(qm, qm_dc_portal_fman0, 1, 0);
-	qm_set_dc(qm, qm_dc_portal_fman1, 1, 0);
-	/* Initialise Error Interrupt Handler */
-	ret = __bind_irq(node);
-	if (ret)
-		return ret;
-	return 0;
+#ifdef CONFIG_FSL_DPAA_CHECKING
+	if (fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))
+		return -EINVAL;
+#endif
+	p = get_affine_portal();
+	if (fq_isset(fq, QMAN_FQ_STATE_CHANGING) ||
+	    fq->state == qman_fq_state_retired ||
+	    fq->state == qman_fq_state_oos) {
+		ret = -EBUSY;
+		goto out;
+	}
+	mcc = qm_mc_start(&p->p);
+	qm_fqid_set(&mcc->fq, fq->fqid);
+	qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE);
+	if (!qm_mc_result_timeout(&p->p, &mcr)) {
+		dev_crit(p->config->dev, "ALTER_RETIRE timeout\n");
+		ret = -ETIMEDOUT;
+		goto out;
+	}
+
+	DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_RETIRE);
+	res = mcr->result;
+	/*
+	 * "Elegant" would be to treat OK/PENDING the same way; set CHANGING,
+	 * and defer the flags until FQRNI or FQRN (respectively) show up. But
+	 * "Friendly" is to process OK immediately, and not set CHANGING. We do
+	 * friendly, otherwise the caller doesn't necessarily have a fully
+	 * "retired" FQ on return even if the retirement was immediate. However
+	 * this does mean some code duplication between here and
+	 * fq_state_change().
+	 */
+	if (res == QM_MCR_RESULT_OK) {
+		ret = 0;
+		/* Process 'fq' right away, we'll ignore FQRNI */
+		if (mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY)
+			fq_set(fq, QMAN_FQ_STATE_NE);
+		if (mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)
+			fq_set(fq, QMAN_FQ_STATE_ORL);
+		if (flags)
+			*flags = fq->flags;
+		fq->state = qman_fq_state_retired;
+		if (fq->cb.fqs) {
+			/*
+			 * Another issue with supporting "immediate" retirement
+			 * is that we're forced to drop FQRNIs, because by the
+			 * time they're seen it may already be "too late" (the
+			 * fq may have been OOS'd and free()'d already). But if
+			 * the upper layer wants a callback whether it's
+			 * immediate or not, we have to fake a "MR" entry to
+			 * look like an FQRNI...
+			 */
+			union qm_mr_entry msg;
+
+			msg.verb = QM_MR_VERB_FQRNI;
+			msg.fq.fqs = mcr->alterfq.fqs;
+			qm_fqid_set(&msg.fq, fq->fqid);
+			msg.fq.context_b = cpu_to_be32(fq_to_tag(fq));
+			fq->cb.fqs(p, fq, &msg);
+		}
+	} else if (res == QM_MCR_RESULT_PENDING) {
+		ret = 1;
+		fq_set(fq, QMAN_FQ_STATE_CHANGING);
+	} else {
+		ret = -EIO;
+	}
+out:
+	put_affine_portal();
+	return ret;
 }
+EXPORT_SYMBOL(qman_retire_fq);
 
-#define LIO_CFG_LIODN_MASK 0x0fff0000
-void qman_liodn_fixup(u16 channel)
+int qman_oos_fq(struct qman_fq *fq)
 {
-	static int done;
-	static u32 liodn_offset;
-	u32 before, after;
-	int idx = channel - QM_CHANNEL_SWPORTAL0;
+	union qm_mc_command *mcc;
+	union qm_mc_result *mcr;
+	struct qman_portal *p;
+	int ret = 0;
 
-	if (!qman_have_ccsr())
-		return;
-	if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
-		before = qm_in(REV3_QCSP_LIO_CFG(idx));
-	else
-		before = qm_in(QCSP_LIO_CFG(idx));
-	if (!done) {
-		liodn_offset = before & LIO_CFG_LIODN_MASK;
-		done = 1;
-		return;
+	if (fq->state != qman_fq_state_retired)
+		return -EINVAL;
+#ifdef CONFIG_FSL_DPAA_CHECKING
+	if (fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))
+		return -EINVAL;
+#endif
+	p = get_affine_portal();
+	if (fq_isset(fq, QMAN_FQ_STATE_BLOCKOOS) ||
+	    fq->state != qman_fq_state_retired) {
+		ret = -EBUSY;
+		goto out;
+	}
+	mcc = qm_mc_start(&p->p);
+	qm_fqid_set(&mcc->fq, fq->fqid);
+	qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS);
+	if (!qm_mc_result_timeout(&p->p, &mcr)) {
+		ret = -ETIMEDOUT;
+		goto out;
 	}
-	after = (before & (~LIO_CFG_LIODN_MASK)) | liodn_offset;
-	if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
-		qm_out(REV3_QCSP_LIO_CFG(idx), after);
+	DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_OOS);
+	if (mcr->result != QM_MCR_RESULT_OK) {
+		ret = -EIO;
+		goto out;
+	}
+	fq->state = qman_fq_state_oos;
+out:
+	put_affine_portal();
+	return ret;
+}
+EXPORT_SYMBOL(qman_oos_fq);
+
+int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd)
+{
+	union qm_mc_command *mcc;
+	union qm_mc_result *mcr;
+	struct qman_portal *p = get_affine_portal();
+	int ret = 0;
+
+	mcc = qm_mc_start(&p->p);
+	qm_fqid_set(&mcc->fq, fq->fqid);
+	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
+	if (!qm_mc_result_timeout(&p->p, &mcr)) {
+		ret = -ETIMEDOUT;
+		goto out;
+	}
+
+	DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ);
+	if (mcr->result == QM_MCR_RESULT_OK)
+		*fqd = mcr->queryfq.fqd;
 	else
-		qm_out(QCSP_LIO_CFG(idx), after);
+		ret = -EIO;
+out:
+	put_affine_portal();
+	return ret;
 }
 
-#define IO_CFG_SDEST_MASK 0x00ff0000
-int qman_set_sdest(u16 channel, unsigned int cpu_idx)
+static int qman_query_fq_np(struct qman_fq *fq,
+			    struct qm_mcr_queryfq_np *np)
 {
-	int idx = channel - QM_CHANNEL_SWPORTAL0;
-	u32 before, after;
+	union qm_mc_command *mcc;
+	union qm_mc_result *mcr;
+	struct qman_portal *p = get_affine_portal();
+	int ret = 0;
+
+	mcc = qm_mc_start(&p->p);
+	qm_fqid_set(&mcc->fq, fq->fqid);
+	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
+	if (!qm_mc_result_timeout(&p->p, &mcr)) {
+		ret = -ETIMEDOUT;
+		goto out;
+	}
 
-	if (!qman_have_ccsr())
-		return -ENODEV;
+	DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
+	if (mcr->result == QM_MCR_RESULT_OK)
+		*np = mcr->queryfq_np;
+	else if (mcr->result == QM_MCR_RESULT_ERR_FQID)
+		ret = -ERANGE;
+	else
+		ret = -EIO;
+out:
+	put_affine_portal();
+	return ret;
+}
 
-	if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) {
-		before = qm_in(REV3_QCSP_IO_CFG(idx));
-		/* Each pair of vcpu share the same SRQ(SDEST) */
-		cpu_idx /= 2;
-		after = (before & (~IO_CFG_SDEST_MASK)) | (cpu_idx << 16);
-		qm_out(REV3_QCSP_IO_CFG(idx), after);
-	} else {
-		before = qm_in(QCSP_IO_CFG(idx));
-		after = (before & (~IO_CFG_SDEST_MASK)) | (cpu_idx << 16);
-		qm_out(QCSP_IO_CFG(idx), after);
+static int qman_query_cgr(struct qman_cgr *cgr,
+			  struct qm_mcr_querycgr *cgrd)
+{
+	union qm_mc_command *mcc;
+	union qm_mc_result *mcr;
+	struct qman_portal *p = get_affine_portal();
+	int ret = 0;
+
+	mcc = qm_mc_start(&p->p);
+	mcc->cgr.cgid = cgr->cgrid;
+	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCGR);
+	if (!qm_mc_result_timeout(&p->p, &mcr)) {
+		ret = -ETIMEDOUT;
+		goto out;
 	}
+	DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYCGR);
+	if (mcr->result == QM_MCR_RESULT_OK)
+		*cgrd = mcr->querycgr;
+	else {
+		dev_err(p->config->dev, "QUERY_CGR failed: %s\n",
+			mcr_result_str(mcr->result));
+		ret = -EIO;
+	}
+out:
+	put_affine_portal();
+	return ret;
+}
+
+int qman_query_cgr_congested(struct qman_cgr *cgr, bool *result)
+{
+	struct qm_mcr_querycgr query_cgr;
+	int err;
+
+	err = qman_query_cgr(cgr, &query_cgr);
+	if (err)
+		return err;
+
+	*result = !!query_cgr.cgr.cs;
 	return 0;
 }
+EXPORT_SYMBOL(qman_query_cgr_congested);
+
+/* internal function used as a wait_event() expression */
+static int set_p_vdqcr(struct qman_portal *p, struct qman_fq *fq, u32 vdqcr)
+{
+	unsigned long irqflags;
+	int ret = -EBUSY;
+
+	local_irq_save(irqflags);
+	if (p->vdqcr_owned)
+		goto out;
+	if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
+		goto out;
+
+	fq_set(fq, QMAN_FQ_STATE_VDQCR);
+	p->vdqcr_owned = fq;
+	qm_dqrr_vdqcr_set(&p->p, vdqcr);
+	ret = 0;
+out:
+	local_irq_restore(irqflags);
+	return ret;
+}
+
+static int set_vdqcr(struct qman_portal **p, struct qman_fq *fq, u32 vdqcr)
+{
+	int ret;
+
+	*p = get_affine_portal();
+	ret = set_p_vdqcr(*p, fq, vdqcr);
+	put_affine_portal();
+	return ret;
+}
 
-#define MISC_CFG_WPM_MASK 0x00000002
-int qm_set_wpm(int wpm)
+static int wait_vdqcr_start(struct qman_portal **p, struct qman_fq *fq,
+				u32 vdqcr, u32 flags)
 {
-	u32 before;
-	u32 after;
+	int ret = 0;
 
-	if (!qman_have_ccsr())
-		return -ENODEV;
+	if (flags & QMAN_VOLATILE_FLAG_WAIT_INT)
+		ret = wait_event_interruptible(affine_queue,
+				!set_vdqcr(p, fq, vdqcr));
+	else
+		wait_event(affine_queue, !set_vdqcr(p, fq, vdqcr));
+	return ret;
+}
+
+int qman_volatile_dequeue(struct qman_fq *fq, u32 flags, u32 vdqcr)
+{
+	struct qman_portal *p;
+	int ret;
 
-	before = qm_in(MISC_CFG);
-	after = (before & (~MISC_CFG_WPM_MASK)) | (wpm << 1);
-	qm_out(MISC_CFG, after);
+	if (fq->state != qman_fq_state_parked &&
+	    fq->state != qman_fq_state_retired)
+		return -EINVAL;
+	if (vdqcr & QM_VDQCR_FQID_MASK)
+		return -EINVAL;
+	if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
+		return -EBUSY;
+	vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid;
+	if (flags & QMAN_VOLATILE_FLAG_WAIT)
+		ret = wait_vdqcr_start(&p, fq, vdqcr, flags);
+	else
+		ret = set_vdqcr(&p, fq, vdqcr);
+	if (ret)
+		return ret;
+	/* VDQCR is set */
+	if (flags & QMAN_VOLATILE_FLAG_FINISH) {
+		if (flags & QMAN_VOLATILE_FLAG_WAIT_INT)
+			/*
+			 * NB: don't propagate any error - the caller wouldn't
+			 * know whether the VDQCR was issued or not. A signal
+			 * could arrive after returning anyway, so the caller
+			 * can check signal_pending() if that's an issue.
+			 */
+			wait_event_interruptible(affine_queue,
+				!fq_isset(fq, QMAN_FQ_STATE_VDQCR));
+		else
+			wait_event(affine_queue,
+				!fq_isset(fq, QMAN_FQ_STATE_VDQCR));
+	}
 	return 0;
 }
+EXPORT_SYMBOL(qman_volatile_dequeue);
+
+static void update_eqcr_ci(struct qman_portal *p, u8 avail)
+{
+	if (avail)
+		qm_eqcr_cce_prefetch(&p->p);
+	else
+		qm_eqcr_cce_update(&p->p);
+}
 
-int qm_get_wpm(int *wpm)
+int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd)
 {
-	u32 before;
+	struct qman_portal *p;
+	struct qm_eqcr_entry *eq;
+	unsigned long irqflags;
+	u8 avail;
+
+	p = get_affine_portal();
+	local_irq_save(irqflags);
+
+	if (p->use_eqcr_ci_stashing) {
+		/*
+		 * The stashing case is easy, only update if we need to in
+		 * order to try and liberate ring entries.
+		 */
+		eq = qm_eqcr_start_stash(&p->p);
+	} else {
+		/*
+		 * The non-stashing case is harder, need to prefetch ahead of
+		 * time.
+		 */
+		avail = qm_eqcr_get_avail(&p->p);
+		if (avail < 2)
+			update_eqcr_ci(p, avail);
+		eq = qm_eqcr_start_no_stash(&p->p);
+	}
+
+	if (unlikely(!eq))
+		goto out;
 
-	if (!qman_have_ccsr())
-		return -ENODEV;
+	qm_fqid_set(eq, fq->fqid);
+	eq->tag = cpu_to_be32(fq_to_tag(fq));
+	eq->fd = *fd;
 
-	before = qm_in(MISC_CFG);
-	*wpm = (before & MISC_CFG_WPM_MASK) >> 1;
+	qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE);
+out:
+	local_irq_restore(irqflags);
+	put_affine_portal();
 	return 0;
 }
+EXPORT_SYMBOL(qman_enqueue);
+
+static int qm_modify_cgr(struct qman_cgr *cgr, u32 flags,
+			 struct qm_mcc_initcgr *opts)
+{
+	union qm_mc_command *mcc;
+	union qm_mc_result *mcr;
+	struct qman_portal *p = get_affine_portal();
+	u8 verb = QM_MCC_VERB_MODIFYCGR;
+	int ret = 0;
+
+	mcc = qm_mc_start(&p->p);
+	if (opts)
+		mcc->initcgr = *opts;
+	mcc->initcgr.cgid = cgr->cgrid;
+	if (flags & QMAN_CGR_FLAG_USE_INIT)
+		verb = QM_MCC_VERB_INITCGR;
+	qm_mc_commit(&p->p, verb);
+	if (!qm_mc_result_timeout(&p->p, &mcr)) {
+		ret = -ETIMEDOUT;
+		goto out;
+	}
+
+	DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == verb);
+	if (mcr->result != QM_MCR_RESULT_OK)
+		ret = -EIO;
 
-#ifdef CONFIG_SYSFS
+out:
+	put_affine_portal();
+	return ret;
+}
 
-#define DRV_NAME	"fsl-qman"
+#define PORTAL_IDX(n)	(n->config->channel - QM_CHANNEL_SWPORTAL0)
 
-static ssize_t show_pfdr_fpc(struct device *dev,
-	struct device_attribute *dev_attr, char *buf)
+/* congestion state change notification target update control */
+static void qm_cgr_cscn_targ_set(struct __qm_mc_cgr *cgr, int pi, u32 val)
 {
-	return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(PFDR_FPC));
-};
+	if (qman_ip_rev >= QMAN_REV30)
+		cgr->cscn_targ_upd_ctrl = cpu_to_be16(pi |
+					QM_CGR_TARG_UDP_CTRL_WRITE_BIT);
+	else
+		cgr->cscn_targ = cpu_to_be32(val | QM_CGR_TARG_PORTAL(pi));
+}
 
-static ssize_t show_dlm_avg(struct device *dev,
-	struct device_attribute *dev_attr, char *buf)
+static void qm_cgr_cscn_targ_clear(struct __qm_mc_cgr *cgr, int pi, u32 val)
 {
-	u32 data;
-	int i;
+	if (qman_ip_rev >= QMAN_REV30)
+		cgr->cscn_targ_upd_ctrl = cpu_to_be16(pi);
+	else
+		cgr->cscn_targ = cpu_to_be32(val & ~QM_CGR_TARG_PORTAL(pi));
+}
 
-	if (sscanf(dev_attr->attr.name, "dcp%d_dlm_avg", &i) != 1)
-		return -EINVAL;
-	data = qm_in(DCP_DLM_AVG(i));
-	return snprintf(buf, PAGE_SIZE, "%d.%08d\n", data>>8,
-			(data & 0x000000ff)*390625);
-};
+static u8 qman_cgr_cpus[CGR_NUM];
 
-static ssize_t set_dlm_avg(struct device *dev,
-	struct device_attribute *dev_attr, const char *buf, size_t count)
+void qman_init_cgr_all(void)
 {
-	unsigned long val;
-	int i;
+	struct qman_cgr cgr;
+	int err_cnt = 0;
 
-	if (sscanf(dev_attr->attr.name, "dcp%d_dlm_avg", &i) != 1)
-		return -EINVAL;
-	if (kstrtoul(buf, 0, &val)) {
-		dev_dbg(dev, "invalid input %s\n", buf);
+	for (cgr.cgrid = 0; cgr.cgrid < CGR_NUM; cgr.cgrid++) {
+		if (qm_modify_cgr(&cgr, QMAN_CGR_FLAG_USE_INIT, NULL))
+			err_cnt++;
+	}
+
+	if (err_cnt)
+		pr_err("Warning: %d error%s while initialising CGR h/w\n",
+		       err_cnt, (err_cnt > 1) ? "s" : "");
+}
+
+int qman_create_cgr(struct qman_cgr *cgr, u32 flags,
+		    struct qm_mcc_initcgr *opts)
+{
+	struct qm_mcr_querycgr cgr_state;
+	int ret;
+	struct qman_portal *p;
+
+	/*
+	 * We have to check that the provided CGRID is within the limits of the
+	 * data-structures, for obvious reasons. However we'll let h/w take
+	 * care of determining whether it's within the limits of what exists on
+	 * the SoC.
+	 */
+	if (cgr->cgrid >= CGR_NUM)
 		return -EINVAL;
+
+	preempt_disable();
+	p = get_affine_portal();
+	qman_cgr_cpus[cgr->cgrid] = smp_processor_id();
+	preempt_enable();
+
+	cgr->chan = p->config->channel;
+	spin_lock(&p->cgr_lock);
+
+	if (opts) {
+		struct qm_mcc_initcgr local_opts = *opts;
+
+		ret = qman_query_cgr(cgr, &cgr_state);
+		if (ret)
+			goto out;
+
+		qm_cgr_cscn_targ_set(&local_opts.cgr, PORTAL_IDX(p),
+				     be32_to_cpu(cgr_state.cgr.cscn_targ));
+		local_opts.we_mask |= cpu_to_be16(QM_CGR_WE_CSCN_TARG);
+
+		/* send init if flags indicate so */
+		if (flags & QMAN_CGR_FLAG_USE_INIT)
+			ret = qm_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT,
+					    &local_opts);
+		else
+			ret = qm_modify_cgr(cgr, 0, &local_opts);
+		if (ret)
+			goto out;
 	}
-	qm_out(DCP_DLM_AVG(i), val);
-	return count;
-};
 
-static ssize_t show_pfdr_cfg(struct device *dev,
-	struct device_attribute *dev_attr, char *buf)
+	list_add(&cgr->node, &p->cgr_cbs);
+
+	/* Determine if newly added object requires its callback to be called */
+	ret = qman_query_cgr(cgr, &cgr_state);
+	if (ret) {
+		/* we can't go back, so proceed and return success */
+		dev_err(p->config->dev, "CGR HW state partially modified\n");
+		ret = 0;
+		goto out;
+	}
+	if (cgr->cb && cgr_state.cgr.cscn_en &&
+	    qman_cgrs_get(&p->cgrs[1], cgr->cgrid))
+		cgr->cb(p, cgr, 1);
+out:
+	spin_unlock(&p->cgr_lock);
+	put_affine_portal();
+	return ret;
+}
+EXPORT_SYMBOL(qman_create_cgr);
+
+int qman_delete_cgr(struct qman_cgr *cgr)
 {
-	return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(PFDR_CFG));
+	unsigned long irqflags;
+	struct qm_mcr_querycgr cgr_state;
+	struct qm_mcc_initcgr local_opts;
+	int ret = 0;
+	struct qman_cgr *i;
+	struct qman_portal *p = get_affine_portal();
+
+	if (cgr->chan != p->config->channel) {
+		/* attempt to delete from other portal than creator */
+		dev_err(p->config->dev, "CGR not owned by current portal");
+		dev_dbg(p->config->dev, " create 0x%x, delete 0x%x\n",
+			cgr->chan, p->config->channel);
+
+		ret = -EINVAL;
+		goto put_portal;
+	}
+	memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
+	spin_lock_irqsave(&p->cgr_lock, irqflags);
+	list_del(&cgr->node);
+	/*
+	 * If there are no other CGR objects for this CGRID in the list,
+	 * update CSCN_TARG accordingly
+	 */
+	list_for_each_entry(i, &p->cgr_cbs, node)
+		if (i->cgrid == cgr->cgrid && i->cb)
+			goto release_lock;
+	ret = qman_query_cgr(cgr, &cgr_state);
+	if (ret)  {
+		/* add back to the list */
+		list_add(&cgr->node, &p->cgr_cbs);
+		goto release_lock;
+	}
+
+	local_opts.we_mask = cpu_to_be16(QM_CGR_WE_CSCN_TARG);
+	qm_cgr_cscn_targ_clear(&local_opts.cgr, PORTAL_IDX(p),
+			       be32_to_cpu(cgr_state.cgr.cscn_targ));
+
+	ret = qm_modify_cgr(cgr, 0, &local_opts);
+	if (ret)
+		/* add back to the list */
+		list_add(&cgr->node, &p->cgr_cbs);
+release_lock:
+	spin_unlock_irqrestore(&p->cgr_lock, irqflags);
+put_portal:
+	put_affine_portal();
+	return ret;
+}
+EXPORT_SYMBOL(qman_delete_cgr);
+
+struct cgr_comp {
+	struct qman_cgr *cgr;
+	struct completion completion;
 };
 
-static ssize_t set_pfdr_cfg(struct device *dev,
-	struct device_attribute *dev_attr, const char *buf, size_t count)
+static int qman_delete_cgr_thread(void *p)
 {
-	unsigned long val;
+	struct cgr_comp *cgr_comp = (struct cgr_comp *)p;
+	int ret;
 
-	if (kstrtoul(buf, 0, &val)) {
-		dev_dbg(dev, "invalid input %s\n", buf);
-		return -EINVAL;
-	}
-	qm_out(PFDR_CFG, val);
-	return count;
-};
+	ret = qman_delete_cgr(cgr_comp->cgr);
+	complete(&cgr_comp->completion);
 
-static ssize_t show_sfdr_in_use(struct device *dev,
-	struct device_attribute *dev_attr, char *buf)
+	return ret;
+}
+
+void qman_delete_cgr_safe(struct qman_cgr *cgr)
 {
-	return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(SFDR_IN_USE));
-};
+	struct task_struct *thread;
+	struct cgr_comp cgr_comp;
+
+	preempt_disable();
+	if (qman_cgr_cpus[cgr->cgrid] != smp_processor_id()) {
+		init_completion(&cgr_comp.completion);
+		cgr_comp.cgr = cgr;
+		thread = kthread_create(qman_delete_cgr_thread, &cgr_comp,
+					"cgr_del");
+
+		if (IS_ERR(thread))
+			goto out;
+
+		kthread_bind(thread, qman_cgr_cpus[cgr->cgrid]);
+		wake_up_process(thread);
+		wait_for_completion(&cgr_comp.completion);
+		preempt_enable();
+		return;
+	}
+out:
+	qman_delete_cgr(cgr);
+	preempt_enable();
+}
+EXPORT_SYMBOL(qman_delete_cgr_safe);
+
+/* Cleanup FQs */
 
-static ssize_t show_idle_stat(struct device *dev,
-	struct device_attribute *dev_attr, char *buf)
+static int _qm_mr_consume_and_match_verb(struct qm_portal *p, int v)
 {
-	return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(IDLE_STAT));
-};
+	const union qm_mr_entry *msg;
+	int found = 0;
+
+	qm_mr_pvb_update(p);
+	msg = qm_mr_current(p);
+	while (msg) {
+		if ((msg->verb & QM_MR_VERB_TYPE_MASK) == v)
+			found = 1;
+		qm_mr_next(p);
+		qm_mr_cci_consume_to_current(p);
+		qm_mr_pvb_update(p);
+		msg = qm_mr_current(p);
+	}
+	return found;
+}
 
-static ssize_t show_ci_rlm_avg(struct device *dev,
-	struct device_attribute *dev_attr, char *buf)
+static int _qm_dqrr_consume_and_match(struct qm_portal *p, u32 fqid, int s,
+				      bool wait)
 {
-	u32 data = qm_in(CI_RLM_AVG);
+	const struct qm_dqrr_entry *dqrr;
+	int found = 0;
 
-	return snprintf(buf, PAGE_SIZE, "%d.%08d\n", data>>8,
-			(data & 0x000000ff)*390625);
-};
+	do {
+		qm_dqrr_pvb_update(p);
+		dqrr = qm_dqrr_current(p);
+		if (!dqrr)
+			cpu_relax();
+	} while (wait && !dqrr);
+
+	while (dqrr) {
+		if (qm_fqid_get(dqrr) == fqid && (dqrr->stat & s))
+			found = 1;
+		qm_dqrr_cdc_consume_1ptr(p, dqrr, 0);
+		qm_dqrr_pvb_update(p);
+		qm_dqrr_next(p);
+		dqrr = qm_dqrr_current(p);
+	}
+	return found;
+}
+
+#define qm_mr_drain(p, V) \
+	_qm_mr_consume_and_match_verb(p, QM_MR_VERB_##V)
+
+#define qm_dqrr_drain(p, f, S) \
+	_qm_dqrr_consume_and_match(p, f, QM_DQRR_STAT_##S, false)
 
-static ssize_t set_ci_rlm_avg(struct device *dev,
-	struct device_attribute *dev_attr, const char *buf, size_t count)
+#define qm_dqrr_drain_wait(p, f, S) \
+	_qm_dqrr_consume_and_match(p, f, QM_DQRR_STAT_##S, true)
+
+#define qm_dqrr_drain_nomatch(p) \
+	_qm_dqrr_consume_and_match(p, 0, 0, false)
+
+static int qman_shutdown_fq(u32 fqid)
 {
-	unsigned long val;
+	struct qman_portal *p;
+#ifndef __rtems__
+	struct device *dev;
+#endif /* __rtems__ */
+	union qm_mc_command *mcc;
+	union qm_mc_result *mcr;
+	int orl_empty, drain = 0, ret = 0;
+	u32 channel, wq, res;
+	u8 state;
 
-	if (kstrtoul(buf, 0, &val)) {
-		dev_dbg(dev, "invalid input %s\n", buf);
-		return -EINVAL;
+	p = get_affine_portal();
+#ifndef __rtems__
+	dev = p->config->dev;
+#endif /* __rtems__ */
+	/* Determine the state of the FQID */
+	mcc = qm_mc_start(&p->p);
+	qm_fqid_set(&mcc->fq, fqid);
+	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
+	if (!qm_mc_result_timeout(&p->p, &mcr)) {
+		dev_err(dev, "QUERYFQ_NP timeout\n");
+		ret = -ETIMEDOUT;
+		goto out;
 	}
-	qm_out(CI_RLM_AVG, val);
-	return count;
-};
 
-static ssize_t show_err_isr(struct device *dev,
-	struct device_attribute *dev_attr, char *buf)
+	DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
+	state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK;
+	if (state == QM_MCR_NP_STATE_OOS)
+		goto out; /* Already OOS, no need to do anymore checks */
+
+	/* Query which channel the FQ is using */
+	mcc = qm_mc_start(&p->p);
+	qm_fqid_set(&mcc->fq, fqid);
+	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
+	if (!qm_mc_result_timeout(&p->p, &mcr)) {
+		dev_err(dev, "QUERYFQ timeout\n");
+		ret = -ETIMEDOUT;
+		goto out;
+	}
+
+	DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ);
+	/* Need to store these since the MCR gets reused */
+	channel = qm_fqd_get_chan(&mcr->queryfq.fqd);
+	wq = qm_fqd_get_wq(&mcr->queryfq.fqd);
+
+	switch (state) {
+	case QM_MCR_NP_STATE_TEN_SCHED:
+	case QM_MCR_NP_STATE_TRU_SCHED:
+	case QM_MCR_NP_STATE_ACTIVE:
+	case QM_MCR_NP_STATE_PARKED:
+		orl_empty = 0;
+		mcc = qm_mc_start(&p->p);
+		qm_fqid_set(&mcc->fq, fqid);
+		qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE);
+		if (!qm_mc_result_timeout(&p->p, &mcr)) {
+			dev_err(dev, "QUERYFQ_NP timeout\n");
+			ret = -ETIMEDOUT;
+			goto out;
+		}
+		DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
+			    QM_MCR_VERB_ALTER_RETIRE);
+		res = mcr->result; /* Make a copy as we reuse MCR below */
+
+		if (res == QM_MCR_RESULT_PENDING) {
+			/*
+			 * Need to wait for the FQRN in the message ring, which
+			 * will only occur once the FQ has been drained.  In
+			 * order for the FQ to drain the portal needs to be set
+			 * to dequeue from the channel the FQ is scheduled on
+			 */
+			int found_fqrn = 0;
+			u16 dequeue_wq = 0;
+
+			/* Flag that we need to drain FQ */
+			drain = 1;
+
+			if (channel >= qm_channel_pool1 &&
+			    channel < qm_channel_pool1 + 15) {
+				/* Pool channel, enable the bit in the portal */
+				dequeue_wq = (channel -
+					      qm_channel_pool1 + 1)<<4 | wq;
+			} else if (channel < qm_channel_pool1) {
+				/* Dedicated channel */
+				dequeue_wq = wq;
+			} else {
+				dev_err(dev, "Can't recover FQ 0x%x, ch: 0x%x",
+					fqid, channel);
+				ret = -EBUSY;
+				goto out;
+			}
+#ifdef __rtems__
+			(void)dequeue_wq;
+#endif /* __rtems__ */
+			/* Set the sdqcr to drain this channel */
+			if (channel < qm_channel_pool1)
+				qm_dqrr_sdqcr_set(&p->p,
+						  QM_SDQCR_TYPE_ACTIVE |
+						  QM_SDQCR_CHANNELS_DEDICATED);
+			else
+				qm_dqrr_sdqcr_set(&p->p,
+						  QM_SDQCR_TYPE_ACTIVE |
+						  QM_SDQCR_CHANNELS_POOL_CONV
+						  (channel));
+			do {
+				/* Keep draining DQRR while checking the MR*/
+				qm_dqrr_drain_nomatch(&p->p);
+				/* Process message ring too */
+				found_fqrn = qm_mr_drain(&p->p, FQRN);
+				cpu_relax();
+			} while (!found_fqrn);
+
+		}
+		if (res != QM_MCR_RESULT_OK &&
+		    res != QM_MCR_RESULT_PENDING) {
+			dev_err(dev, "retire_fq failed: FQ 0x%x, res=0x%x\n",
+				fqid, res);
+			ret = -EIO;
+			goto out;
+		}
+		if (!(mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)) {
+			/*
+			 * ORL had no entries, no need to wait until the
+			 * ERNs come in
+			 */
+			orl_empty = 1;
+		}
+		/*
+		 * Retirement succeeded, check to see if FQ needs
+		 * to be drained
+		 */
+		if (drain || mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) {
+			/* FQ is Not Empty, drain using volatile DQ commands */
+			do {
+				u32 vdqcr = fqid | QM_VDQCR_NUMFRAMES_SET(3);
+
+				qm_dqrr_vdqcr_set(&p->p, vdqcr);
+				/*
+				 * Wait for a dequeue and process the dequeues,
+				 * making sure to empty the ring completely
+				 */
+			} while (qm_dqrr_drain_wait(&p->p, fqid, FQ_EMPTY));
+		}
+		qm_dqrr_sdqcr_set(&p->p, 0);
+
+		while (!orl_empty) {
+			/* Wait for the ORL to have been completely drained */
+			orl_empty = qm_mr_drain(&p->p, FQRL);
+			cpu_relax();
+		}
+		mcc = qm_mc_start(&p->p);
+		qm_fqid_set(&mcc->fq, fqid);
+		qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS);
+		if (!qm_mc_result_timeout(&p->p, &mcr)) {
+			ret = -ETIMEDOUT;
+			goto out;
+		}
+
+		DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
+			    QM_MCR_VERB_ALTER_OOS);
+		if (mcr->result != QM_MCR_RESULT_OK) {
+			dev_err(dev, "OOS after drain fail: FQ 0x%x (0x%x)\n",
+				fqid, mcr->result);
+			ret = -EIO;
+			goto out;
+		}
+		break;
+
+	case QM_MCR_NP_STATE_RETIRED:
+		/* Send OOS Command */
+		mcc = qm_mc_start(&p->p);
+		qm_fqid_set(&mcc->fq, fqid);
+		qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS);
+		if (!qm_mc_result_timeout(&p->p, &mcr)) {
+			ret = -ETIMEDOUT;
+			goto out;
+		}
+
+		DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
+			    QM_MCR_VERB_ALTER_OOS);
+		if (mcr->result) {
+			dev_err(dev, "OOS fail: FQ 0x%x (0x%x)\n",
+				fqid, mcr->result);
+			ret = -EIO;
+			goto out;
+		}
+		break;
+
+	case QM_MCR_NP_STATE_OOS:
+		/*  Done */
+		break;
+
+	default:
+		ret = -EIO;
+	}
+
+out:
+	put_affine_portal();
+	return ret;
+}
+
+const struct qm_portal_config *qman_get_qm_portal_config(
+						struct qman_portal *portal)
 {
-	return snprintf(buf, PAGE_SIZE, "0x%08x\n", qm_in(ERR_ISR));
-};
+	return portal->config;
+}
+EXPORT_SYMBOL(qman_get_qm_portal_config);
 
+struct gen_pool *qm_fqalloc; /* FQID allocator */
+struct gen_pool *qm_qpalloc; /* pool-channel allocator */
+struct gen_pool *qm_cgralloc; /* CGR ID allocator */
 
-static ssize_t show_sbec(struct device *dev,
-	struct device_attribute *dev_attr, char *buf)
+static int qman_alloc_range(struct gen_pool *p, u32 *result, u32 cnt)
 {
-	int i;
+	unsigned long addr;
 
-	if (sscanf(dev_attr->attr.name, "sbec_%d", &i) != 1)
-		return -EINVAL;
-	return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(SBEC(i)));
-};
+	addr = gen_pool_alloc(p, cnt);
+	if (!addr)
+		return -ENOMEM;
 
-static DEVICE_ATTR(pfdr_fpc, S_IRUSR, show_pfdr_fpc, NULL);
-static DEVICE_ATTR(pfdr_cfg, S_IRUSR, show_pfdr_cfg, set_pfdr_cfg);
-static DEVICE_ATTR(idle_stat, S_IRUSR, show_idle_stat, NULL);
-static DEVICE_ATTR(ci_rlm_avg, (S_IRUSR|S_IWUSR),
-		show_ci_rlm_avg, set_ci_rlm_avg);
-static DEVICE_ATTR(err_isr, S_IRUSR, show_err_isr, NULL);
-static DEVICE_ATTR(sfdr_in_use, S_IRUSR, show_sfdr_in_use, NULL);
-
-static DEVICE_ATTR(dcp0_dlm_avg, (S_IRUSR|S_IWUSR), show_dlm_avg, set_dlm_avg);
-static DEVICE_ATTR(dcp1_dlm_avg, (S_IRUSR|S_IWUSR), show_dlm_avg, set_dlm_avg);
-static DEVICE_ATTR(dcp2_dlm_avg, (S_IRUSR|S_IWUSR), show_dlm_avg, set_dlm_avg);
-static DEVICE_ATTR(dcp3_dlm_avg, (S_IRUSR|S_IWUSR), show_dlm_avg, set_dlm_avg);
-
-static DEVICE_ATTR(sbec_0, S_IRUSR, show_sbec, NULL);
-static DEVICE_ATTR(sbec_1, S_IRUSR, show_sbec, NULL);
-static DEVICE_ATTR(sbec_2, S_IRUSR, show_sbec, NULL);
-static DEVICE_ATTR(sbec_3, S_IRUSR, show_sbec, NULL);
-static DEVICE_ATTR(sbec_4, S_IRUSR, show_sbec, NULL);
-static DEVICE_ATTR(sbec_5, S_IRUSR, show_sbec, NULL);
-static DEVICE_ATTR(sbec_6, S_IRUSR, show_sbec, NULL);
-static DEVICE_ATTR(sbec_7, S_IRUSR, show_sbec, NULL);
-static DEVICE_ATTR(sbec_8, S_IRUSR, show_sbec, NULL);
-static DEVICE_ATTR(sbec_9, S_IRUSR, show_sbec, NULL);
-static DEVICE_ATTR(sbec_10, S_IRUSR, show_sbec, NULL);
-static DEVICE_ATTR(sbec_11, S_IRUSR, show_sbec, NULL);
-static DEVICE_ATTR(sbec_12, S_IRUSR, show_sbec, NULL);
-static DEVICE_ATTR(sbec_13, S_IRUSR, show_sbec, NULL);
-static DEVICE_ATTR(sbec_14, S_IRUSR, show_sbec, NULL);
-
-static struct attribute *qman_dev_attributes[] = {
-	&dev_attr_pfdr_fpc.attr,
-	&dev_attr_pfdr_cfg.attr,
-	&dev_attr_idle_stat.attr,
-	&dev_attr_ci_rlm_avg.attr,
-	&dev_attr_err_isr.attr,
-	&dev_attr_dcp0_dlm_avg.attr,
-	&dev_attr_dcp1_dlm_avg.attr,
-	&dev_attr_dcp2_dlm_avg.attr,
-	&dev_attr_dcp3_dlm_avg.attr,
-	/* sfdr_in_use will be added if necessary */
-	NULL
-};
+	*result = addr & ~DPAA_GENALLOC_OFF;
 
-static struct attribute *qman_dev_ecr_attributes[] = {
-	&dev_attr_sbec_0.attr,
-	&dev_attr_sbec_1.attr,
-	&dev_attr_sbec_2.attr,
-	&dev_attr_sbec_3.attr,
-	&dev_attr_sbec_4.attr,
-	&dev_attr_sbec_5.attr,
-	&dev_attr_sbec_6.attr,
-	&dev_attr_sbec_7.attr,
-	&dev_attr_sbec_8.attr,
-	&dev_attr_sbec_9.attr,
-	&dev_attr_sbec_10.attr,
-	&dev_attr_sbec_11.attr,
-	&dev_attr_sbec_12.attr,
-	&dev_attr_sbec_13.attr,
-	&dev_attr_sbec_14.attr,
-	NULL
-};
+	return 0;
+}
 
-/* root level */
-static const struct attribute_group qman_dev_attr_grp = {
-	.name = NULL,
-	.attrs = qman_dev_attributes
-};
-static const struct attribute_group qman_dev_ecr_grp = {
-	.name = "error_capture",
-	.attrs = qman_dev_ecr_attributes
-};
+int qman_alloc_fqid_range(u32 *result, u32 count)
+{
+	return qman_alloc_range(qm_fqalloc, result, count);
+}
+EXPORT_SYMBOL(qman_alloc_fqid_range);
 
-static int of_fsl_qman_remove(struct platform_device *ofdev)
+int qman_alloc_pool_range(u32 *result, u32 count)
 {
-	sysfs_remove_group(&ofdev->dev.kobj, &qman_dev_attr_grp);
+	return qman_alloc_range(qm_qpalloc, result, count);
+}
+EXPORT_SYMBOL(qman_alloc_pool_range);
+
+int qman_alloc_cgrid_range(u32 *result, u32 count)
+{
+	return qman_alloc_range(qm_cgralloc, result, count);
+}
+EXPORT_SYMBOL(qman_alloc_cgrid_range);
+
+int qman_release_fqid(u32 fqid)
+{
+	int ret = qman_shutdown_fq(fqid);
+
+	if (ret) {
+		pr_debug("FQID %d leaked\n", fqid);
+		return ret;
+	}
+
+	gen_pool_free(qm_fqalloc, fqid | DPAA_GENALLOC_OFF, 1);
 	return 0;
-};
+}
+EXPORT_SYMBOL(qman_release_fqid);
+
+static int qpool_cleanup(u32 qp)
+{
+	/*
+	 * We query all FQDs starting from
+	 * FQID 1 until we get an "invalid FQID" error, looking for non-OOS FQDs
+	 * whose destination channel is the pool-channel being released.
+	 * When a non-OOS FQD is found we attempt to clean it up
+	 */
+	struct qman_fq fq = {
+		.fqid = QM_FQID_RANGE_START
+	};
+	int err;
+
+	do {
+		struct qm_mcr_queryfq_np np;
+
+		err = qman_query_fq_np(&fq, &np);
+		if (err == -ERANGE)
+			/* FQID range exceeded, found no problems */
+			return 0;
+		else if (WARN_ON(err))
+			return err;
+
+		if ((np.state & QM_MCR_NP_STATE_MASK) != QM_MCR_NP_STATE_OOS) {
+			struct qm_fqd fqd;
+
+			err = qman_query_fq(&fq, &fqd);
+			if (WARN_ON(err))
+				return err;
+			if (qm_fqd_get_chan(&fqd) == qp) {
+				/* The channel is the FQ's target, clean it */
+				err = qman_shutdown_fq(fq.fqid);
+				if (err)
+					/*
+					 * Couldn't shut down the FQ
+					 * so the pool must be leaked
+					 */
+					return err;
+			}
+		}
+		/* Move to the next FQID */
+		fq.fqid++;
+	} while (1);
+}
 
-static int of_fsl_qman_probe(struct platform_device *ofdev)
+int qman_release_pool(u32 qp)
 {
 	int ret;
-	struct device *dev = &ofdev->dev;
 
-	ret = sysfs_create_group(&dev->kobj, &qman_dev_attr_grp);
-	if (ret)
-		goto done;
-	ret = sysfs_add_file_to_group(&dev->kobj,
-		&dev_attr_sfdr_in_use.attr, qman_dev_attr_grp.name);
-	if (ret)
-		goto del_group_0;
-	ret = sysfs_create_group(&dev->kobj, &qman_dev_ecr_grp);
-	if (ret)
-		goto del_group_0;
+	ret = qpool_cleanup(qp);
+	if (ret) {
+		pr_debug("CHID %d leaked\n", qp);
+		return ret;
+	}
 
-	goto done;
+	gen_pool_free(qm_qpalloc, qp | DPAA_GENALLOC_OFF, 1);
+	return 0;
+}
+EXPORT_SYMBOL(qman_release_pool);
 
-del_group_0:
-	sysfs_remove_group(&dev->kobj, &qman_dev_attr_grp);
-done:
-	if (ret)
-		dev_err(dev, "Cannot create dev attributes ret=%d\n", ret);
-	return ret;
-};
+static int cgr_cleanup(u32 cgrid)
+{
+	/*
+	 * query all FQDs starting from FQID 1 until we get an "invalid FQID"
+	 * error, looking for non-OOS FQDs whose CGR is the CGR being released
+	 */
+	struct qman_fq fq = {
+		.fqid = QM_FQID_RANGE_START
+	};
+	int err;
 
-static const struct of_device_id of_fsl_qman_ids[] = {
-	{
-		.compatible = "fsl,qman",
-	},
-	{}
-};
+	do {
+		struct qm_mcr_queryfq_np np;
+
+		err = qman_query_fq_np(&fq, &np);
+		if (err == -ERANGE)
+			/* FQID range exceeded, found no problems */
+			return 0;
+		else if (WARN_ON(err))
+			return err;
+
+		if ((np.state & QM_MCR_NP_STATE_MASK) != QM_MCR_NP_STATE_OOS) {
+			struct qm_fqd fqd;
+
+			err = qman_query_fq(&fq, &fqd);
+			if (WARN_ON(err))
+				return err;
+			if (be16_to_cpu(fqd.fq_ctrl) & QM_FQCTRL_CGE &&
+			    fqd.cgid == cgrid) {
+				pr_err("CRGID 0x%x is being used by FQID 0x%x, CGR will be leaked\n",
+				       cgrid, fq.fqid);
+				return -EIO;
+			}
+		}
+		/* Move to the next FQID */
+		fq.fqid++;
+	} while (1);
+}
 
-static struct platform_driver of_fsl_qman_driver = {
-	.driver = {
-		.name = DRV_NAME,
-		.of_match_table = of_fsl_qman_ids,
-	},
-	.probe = of_fsl_qman_probe,
-	.remove	= of_fsl_qman_remove,
-};
+int qman_release_cgrid(u32 cgrid)
+{
+	int ret;
 
-builtin_platform_driver(of_fsl_qman_driver);
+	ret = cgr_cleanup(cgrid);
+	if (ret) {
+		pr_debug("CGRID %d leaked\n", cgrid);
+		return ret;
+	}
 
-#endif /* CONFIG_SYSFS */
+	gen_pool_free(qm_cgralloc, cgrid | DPAA_GENALLOC_OFF, 1);
+	return 0;
+}
+EXPORT_SYMBOL(qman_release_cgrid);