1 files changed, 235 insertions, 493 deletions

diff --git a/freebsd/sys/kern/uipc_mbuf.c b/freebsd/sys/kern/uipc_mbuf.c
index abf4dd3e..db4975ca 100644
--- a/freebsd/sys/kern/uipc_mbuf.c
+++ b/freebsd/sys/kern/uipc_mbuf.c
@@ -49,6 +49,51 @@ __FBSDID("$FreeBSD$");
 #include <sys/domain.h>
 #include <sys/protosw.h>
 #include <sys/uio.h>
+#include <sys/sdt.h>
+
+SDT_PROBE_DEFINE5_XLATE(sdt, , , m__init,
+    "struct mbuf *", "mbufinfo_t *",
+    "uint32_t", "uint32_t",
+    "uint16_t", "uint16_t",
+    "uint32_t", "uint32_t",
+    "uint32_t", "uint32_t");
+
+SDT_PROBE_DEFINE3_XLATE(sdt, , , m__gethdr,
+    "uint32_t", "uint32_t",
+    "uint16_t", "uint16_t",
+    "struct mbuf *", "mbufinfo_t *");
+
+SDT_PROBE_DEFINE3_XLATE(sdt, , , m__get,
+    "uint32_t", "uint32_t",
+    "uint16_t", "uint16_t",
+    "struct mbuf *", "mbufinfo_t *");
+
+SDT_PROBE_DEFINE4_XLATE(sdt, , , m__getcl,
+    "uint32_t", "uint32_t",
+    "uint16_t", "uint16_t",
+    "uint32_t", "uint32_t",
+    "struct mbuf *", "mbufinfo_t *");
+
+SDT_PROBE_DEFINE3_XLATE(sdt, , , m__clget,
+    "struct mbuf *", "mbufinfo_t *",
+    "uint32_t", "uint32_t",
+    "uint32_t", "uint32_t");
+
+SDT_PROBE_DEFINE4_XLATE(sdt, , , m__cljget,
+    "struct mbuf *", "mbufinfo_t *",
+    "uint32_t", "uint32_t",
+    "uint32_t", "uint32_t",
+    "void*", "void*");
+
+SDT_PROBE_DEFINE(sdt, , , m__cljset);
+
+SDT_PROBE_DEFINE1_XLATE(sdt, , , m__free,
+        "struct mbuf *", "mbufinfo_t *");
+
+SDT_PROBE_DEFINE1_XLATE(sdt, , , m__freem,
+    "struct mbuf *", "mbufinfo_t *");
+
+#include <security/mac/mac_framework.h>
 
 int	max_linkhdr;
 int	max_protohdr;
@@ -87,224 +132,93 @@ SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragrandomfailures, CTLFLAG_RW,
 #endif
 
 /*
- * Allocate a given length worth of mbufs and/or clusters (whatever fits
- * best) and return a pointer to the top of the allocated chain.  If an
- * existing mbuf chain is provided, then we will append the new chain
- * to the existing one but still return the top of the newly allocated
- * chain.
+ * Ensure the correct size of various mbuf parameters.  It could be off due
+ * to compiler-induced padding and alignment artifacts.
  */
-struct mbuf *
-m_getm2(struct mbuf *m, int len, int how, short type, int flags)
-{
-	struct mbuf *mb, *nm = NULL, *mtail = NULL;
-
-	KASSERT(len >= 0, ("%s: len is < 0", __func__));
-
-	/* Validate flags. */
-	flags &= (M_PKTHDR | M_EOR);
-
-	/* Packet header mbuf must be first in chain. */
-	if ((flags & M_PKTHDR) && m != NULL)
-		flags &= ~M_PKTHDR;
-
-	/* Loop and append maximum sized mbufs to the chain tail. */
-	while (len > 0) {
-		if (len > MCLBYTES)
-			mb = m_getjcl(how, type, (flags & M_PKTHDR),
-			    MJUMPAGESIZE);
-		else if (len >= MINCLSIZE)
-			mb = m_getcl(how, type, (flags & M_PKTHDR));
-		else if (flags & M_PKTHDR)
-			mb = m_gethdr(how, type);
-		else
-			mb = m_get(how, type);
-
-		/* Fail the whole operation if one mbuf can't be allocated. */
-		if (mb == NULL) {
-			if (nm != NULL)
-				m_freem(nm);
-			return (NULL);
-		}
-
-		/* Book keeping. */
-		len -= (mb->m_flags & M_EXT) ? mb->m_ext.ext_size :
-			((mb->m_flags & M_PKTHDR) ? MHLEN : MLEN);
-		if (mtail != NULL)
-			mtail->m_next = mb;
-		else
-			nm = mb;
-		mtail = mb;
-		flags &= ~M_PKTHDR;	/* Only valid on the first mbuf. */
-	}
-	if (flags & M_EOR)
-		mtail->m_flags |= M_EOR;  /* Only valid on the last mbuf. */
-
-	/* If mbuf was supplied, append new chain to the end of it. */
-	if (m != NULL) {
-		for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next)
-			;
-		mtail->m_next = nm;
-		mtail->m_flags &= ~M_EOR;
-	} else
-		m = nm;
-
-	return (m);
-}
+CTASSERT(MSIZE - offsetof(struct mbuf, m_dat) == MLEN);
+CTASSERT(MSIZE - offsetof(struct mbuf, m_pktdat) == MHLEN);
 
 /*
- * Free an entire chain of mbufs and associated external buffers, if
- * applicable.
+ * mbuf data storage should be 64-bit aligned regardless of architectural
+ * pointer size; check this is the case with and without a packet header.
  */
-void
-m_freem(struct mbuf *mb)
-{
+CTASSERT(offsetof(struct mbuf, m_dat) % 8 == 0);
+CTASSERT(offsetof(struct mbuf, m_pktdat) % 8 == 0);
 
-	while (mb != NULL)
-		mb = m_free(mb);
-}
-
-/*-
- * Configure a provided mbuf to refer to the provided external storage
- * buffer and setup a reference count for said buffer.  If the setting
- * up of the reference count fails, the M_EXT bit will not be set.  If
- * successfull, the M_EXT bit is set in the mbuf's flags.
- *
- * Arguments:
- *    mb     The existing mbuf to which to attach the provided buffer.
- *    buf    The address of the provided external storage buffer.
- *    size   The size of the provided buffer.
- *    freef  A pointer to a routine that is responsible for freeing the
- *           provided external storage buffer.
- *    args   A pointer to an argument structure (of any type) to be passed
- *           to the provided freef routine (may be NULL).
- *    flags  Any other flags to be passed to the provided mbuf.
- *    type   The type that the external storage buffer should be
- *           labeled with.
+/*
+ * While the specific values here don't matter too much (i.e., +/- a few
+ * words), we do want to ensure that changes to these values are carefully
+ * reasoned about and properly documented.  This is especially the case as
+ * network-protocol and device-driver modules encode these layouts, and must
+ * be recompiled if the structures change.  Check these values at compile time
+ * against the ones documented in comments in mbuf.h.
  *
- * Returns:
- *    Nothing.
+ * NB: Possibly they should be documented there via #define's and not just
+ * comments.
  */
-void
-m_extadd(struct mbuf *mb, caddr_t buf, u_int size,
-    void (*freef)(void *, void *), void *arg1, void *arg2, int flags, int type)
-{
-	KASSERT(type != EXT_CLUSTER, ("%s: EXT_CLUSTER not allowed", __func__));
-
-	if (type != EXT_EXTREF)
-		mb->m_ext.ref_cnt = (u_int *)uma_zalloc(zone_ext_refcnt, M_NOWAIT);
-	if (mb->m_ext.ref_cnt != NULL) {
-		*(mb->m_ext.ref_cnt) = 1;
-		mb->m_flags |= (M_EXT | flags);
-		mb->m_ext.ext_buf = buf;
-		mb->m_data = mb->m_ext.ext_buf;
-		mb->m_ext.ext_size = size;
-		mb->m_ext.ext_free = freef;
-		mb->m_ext.ext_arg1 = arg1;
-		mb->m_ext.ext_arg2 = arg2;
-		mb->m_ext.ext_type = type;
-        }
-}
+#if defined(__LP64__)
+CTASSERT(offsetof(struct mbuf, m_dat) == 32);
+CTASSERT(sizeof(struct pkthdr) == 56);
+CTASSERT(sizeof(struct m_ext) == 48);
+#else
+CTASSERT(offsetof(struct mbuf, m_dat) == 24);
+CTASSERT(sizeof(struct pkthdr) == 48);
+CTASSERT(sizeof(struct m_ext) == 28);
+#endif
 
 /*
- * Non-directly-exported function to clean up after mbufs with M_EXT
- * storage attached to them if the reference count hits 1.
+ * Assert that the queue(3) macros produce code of the same size as an old
+ * plain pointer does.
  */
-void
-mb_free_ext(struct mbuf *m)
-{
-	int skipmbuf;
-	
-	KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
-	KASSERT(m->m_ext.ref_cnt != NULL, ("%s: ref_cnt not set", __func__));
-
-
-	/*
-	 * check if the header is embedded in the cluster
-	 */     
-	skipmbuf = (m->m_flags & M_NOFREE);
-	
-	/* Free attached storage if this mbuf is the only reference to it. */
-	if (*(m->m_ext.ref_cnt) == 1 ||
-	    atomic_fetchadd_int(m->m_ext.ref_cnt, -1) == 1) {
-		switch (m->m_ext.ext_type) {
-		case EXT_PACKET:	/* The packet zone is special. */
-			if (*(m->m_ext.ref_cnt) == 0)
-				*(m->m_ext.ref_cnt) = 1;
-			uma_zfree(zone_pack, m);
-			return;		/* Job done. */
-		case EXT_CLUSTER:
-			uma_zfree(zone_clust, m->m_ext.ext_buf);
-			break;
-		case EXT_JUMBOP:
-			uma_zfree(zone_jumbop, m->m_ext.ext_buf);
-			break;
-		case EXT_JUMBO9:
-			uma_zfree(zone_jumbo9, m->m_ext.ext_buf);
-			break;
-		case EXT_JUMBO16:
-			uma_zfree(zone_jumbo16, m->m_ext.ext_buf);
-			break;
-		case EXT_SFBUF:
-		case EXT_NET_DRV:
-		case EXT_MOD_TYPE:
-		case EXT_DISPOSABLE:
-			*(m->m_ext.ref_cnt) = 0;
-			uma_zfree(zone_ext_refcnt, __DEVOLATILE(u_int *,
-				m->m_ext.ref_cnt));
-			/* FALLTHROUGH */
-		case EXT_EXTREF:
-			KASSERT(m->m_ext.ext_free != NULL,
-				("%s: ext_free not set", __func__));
-			(*(m->m_ext.ext_free))(m->m_ext.ext_arg1,
-			    m->m_ext.ext_arg2);
-			break;
-		default:
-			KASSERT(m->m_ext.ext_type == 0,
-				("%s: unknown ext_type", __func__));
-		}
-	}
-	if (skipmbuf)
-		return;
-	
-	/*
-	 * Free this mbuf back to the mbuf zone with all m_ext
-	 * information purged.
-	 */
-	m->m_ext.ext_buf = NULL;
-	m->m_ext.ext_free = NULL;
-	m->m_ext.ext_arg1 = NULL;
-	m->m_ext.ext_arg2 = NULL;
-	m->m_ext.ref_cnt = NULL;
-	m->m_ext.ext_size = 0;
-	m->m_ext.ext_type = 0;
-	m->m_flags &= ~M_EXT;
-	uma_zfree(zone_mbuf, m);
-}
+#ifdef INVARIANTS
+static struct mbuf m_assertbuf;
+CTASSERT(sizeof(m_assertbuf.m_slist) == sizeof(m_assertbuf.m_next));
+CTASSERT(sizeof(m_assertbuf.m_stailq) == sizeof(m_assertbuf.m_next));
+CTASSERT(sizeof(m_assertbuf.m_slistpkt) == sizeof(m_assertbuf.m_nextpkt));
+CTASSERT(sizeof(m_assertbuf.m_stailqpkt) == sizeof(m_assertbuf.m_nextpkt));
+#endif
 
 /*
  * Attach the cluster from *m to *n, set up m_ext in *n
  * and bump the refcount of the cluster.
  */
-static void
+void
 mb_dupcl(struct mbuf *n, struct mbuf *m)
 {
-	KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
-	KASSERT(m->m_ext.ref_cnt != NULL, ("%s: ref_cnt not set", __func__));
-	KASSERT((n->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__));
+	volatile u_int *refcnt;
 
-	if (*(m->m_ext.ref_cnt) == 1)
-		*(m->m_ext.ref_cnt) += 1;
-	else
-		atomic_add_int(m->m_ext.ref_cnt, 1);
-	n->m_ext.ext_buf = m->m_ext.ext_buf;
-	n->m_ext.ext_free = m->m_ext.ext_free;
-	n->m_ext.ext_arg1 = m->m_ext.ext_arg1;
-	n->m_ext.ext_arg2 = m->m_ext.ext_arg2;
-	n->m_ext.ext_size = m->m_ext.ext_size;
-	n->m_ext.ref_cnt = m->m_ext.ref_cnt;
-	n->m_ext.ext_type = m->m_ext.ext_type;
+	KASSERT(m->m_flags & M_EXT, ("%s: M_EXT not set on %p", __func__, m));
+	KASSERT(!(n->m_flags & M_EXT), ("%s: M_EXT set on %p", __func__, n));
+
+	n->m_ext = m->m_ext;
 	n->m_flags |= M_EXT;
 	n->m_flags |= m->m_flags & M_RDONLY;
+
+	/* See if this is the mbuf that holds the embedded refcount. */
+	if (m->m_ext.ext_flags & EXT_FLAG_EMBREF) {
+		refcnt = n->m_ext.ext_cnt = &m->m_ext.ext_count;
+		n->m_ext.ext_flags &= ~EXT_FLAG_EMBREF;
+	} else {
+		KASSERT(m->m_ext.ext_cnt != NULL,
+		    ("%s: no refcounting pointer on %p", __func__, m));
+		refcnt = m->m_ext.ext_cnt;
+	}
+
+	if (*refcnt == 1)
+		*refcnt += 1;
+	else
+		atomic_add_int(refcnt, 1);
+}
+
+void
+m_demote_pkthdr(struct mbuf *m)
+{
+
+	M_ASSERTPKTHDR(m);
+
+	m_tag_delete_chain(m, NULL);
+	m->m_flags &= ~M_PKTHDR;
+	bzero(&m->m_pkthdr, sizeof(struct pkthdr));
 }
 
 /*
@@ -313,23 +227,16 @@ mb_dupcl(struct mbuf *n, struct mbuf *m)
  * cleaned too.
  */
 void
-m_demote(struct mbuf *m0, int all)
+m_demote(struct mbuf *m0, int all, int flags)
 {
 	struct mbuf *m;
 
 	for (m = all ? m0 : m0->m_next; m != NULL; m = m->m_next) {
-		if (m->m_flags & M_PKTHDR) {
-			m_tag_delete_chain(m, NULL);
-			m->m_flags &= ~M_PKTHDR;
-			bzero(&m->m_pkthdr, sizeof(struct pkthdr));
-		}
-		if (m != m0 && m->m_nextpkt != NULL) {
-			KASSERT(m->m_nextpkt == NULL,
-			    ("%s: m_nextpkt not NULL", __func__));
-			m_freem(m->m_nextpkt);
-			m->m_nextpkt = NULL;
-		}
-		m->m_flags = m->m_flags & (M_EXT|M_RDONLY|M_FREELIST|M_NOFREE);
+		KASSERT(m->m_nextpkt == NULL, ("%s: m_nextpkt in m %p, m0 %p",
+		    __func__, m, m0));
+		if (m->m_flags & M_PKTHDR)
+			m_demote_pkthdr(m);
+		m->m_flags = m->m_flags & (M_EXT | M_RDONLY | M_NOFREE | flags);
 	}
 }
 
@@ -349,7 +256,7 @@ m_sanity(struct mbuf *m0, int sanitize)
 
 #ifdef INVARIANTS
 #define	M_SANITY_ACTION(s)	panic("mbuf %p: " s, m)
-#else 
+#else
 #define	M_SANITY_ACTION(s)	printf("mbuf %p: " s, m)
 #endif
 
@@ -359,22 +266,14 @@ m_sanity(struct mbuf *m0, int sanitize)
 		 * unrelated kernel memory before or after us is trashed.
 		 * No way to recover from that.
 		 */
-		a = ((m->m_flags & M_EXT) ? m->m_ext.ext_buf :
-			((m->m_flags & M_PKTHDR) ? (caddr_t)(&m->m_pktdat) :
-			 (caddr_t)(&m->m_dat)) );
-		b = (caddr_t)(a + (m->m_flags & M_EXT ? m->m_ext.ext_size :
-			((m->m_flags & M_PKTHDR) ? MHLEN : MLEN)));
+		a = M_START(m);
+		b = a + M_SIZE(m);
 		if ((caddr_t)m->m_data < a)
 			M_SANITY_ACTION("m_data outside mbuf data range left");
 		if ((caddr_t)m->m_data > b)
 			M_SANITY_ACTION("m_data outside mbuf data range right");
 		if ((caddr_t)m->m_data + m->m_len > b)
 			M_SANITY_ACTION("m_data + m_len exeeds mbuf space");
-		if ((m->m_flags & M_PKTHDR) && m->m_pkthdr.header) {
-			if ((caddr_t)m->m_pkthdr.header < a ||
-			    (caddr_t)m->m_pkthdr.header > b)
-				M_SANITY_ACTION("m_pkthdr.header outside mbuf data range");
-		}
 
 		/* m->m_nextpkt may only be set on first mbuf in chain. */
 		if (m != m0 && m->m_nextpkt != NULL) {
@@ -421,6 +320,26 @@ m_sanity(struct mbuf *m0, int sanitize)
 #undef	M_SANITY_ACTION
 }
 
+/*
+ * Non-inlined part of m_init().
+ */
+int
+m_pkthdr_init(struct mbuf *m, int how)
+{
+#ifdef MAC
+	int error;
+#endif
+	m->m_data = m->m_pktdat;
+	bzero(&m->m_pkthdr, sizeof(m->m_pkthdr));
+#ifdef MAC
+	/* If the label init fails, fail the alloc */
+	error = mac_mbuf_init(m, how);
+	if (error)
+		return (error);
+#endif
+
+	return (0);
+}
 
 /*
  * "Move" mbuf pkthdr from "from" to "to".
@@ -458,14 +377,14 @@ m_move_pkthdr(struct mbuf *to, struct mbuf *from)
  * In particular, this does a deep copy of the packet tags.
  */
 int
-m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how)
+m_dup_pkthdr(struct mbuf *to, const struct mbuf *from, int how)
 {
 
 #if 0
 	/*
 	 * The mbuf allocator only initializes the pkthdr
-	 * when the mbuf is allocated with MGETHDR. Many users
-	 * (e.g. m_copy*, m_prepend) use MGET and then
+	 * when the mbuf is allocated with m_gethdr(). Many users
+	 * (e.g. m_copy*, m_prepend) use m_get() and then
 	 * smash the pkthdr as needed causing these
 	 * assertions to trip.  For now just disable them.
 	 */
@@ -483,7 +402,7 @@ m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how)
 		to->m_data = to->m_pktdat;
 	to->m_pkthdr = from->m_pkthdr;
 	SLIST_INIT(&to->m_pkthdr.tags);
-	return (m_tag_copy_chain(to, from, MBTOM(how)));
+	return (m_tag_copy_chain(to, from, how));
 }
 
 /*
@@ -497,24 +416,19 @@ m_prepend(struct mbuf *m, int len, int how)
 	struct mbuf *mn;
 
 	if (m->m_flags & M_PKTHDR)
-		MGETHDR(mn, how, m->m_type);
+		mn = m_gethdr(how, m->m_type);
 	else
-		MGET(mn, how, m->m_type);
+		mn = m_get(how, m->m_type);
 	if (mn == NULL) {
 		m_freem(m);
 		return (NULL);
 	}
 	if (m->m_flags & M_PKTHDR)
-		M_MOVE_PKTHDR(mn, m);
+		m_move_pkthdr(mn, m);
 	mn->m_next = m;
 	m = mn;
-	if(m->m_flags & M_PKTHDR) {
-		if (len < MHLEN)
-			MH_ALIGN(m, len);
-	} else {
-		if (len < MLEN) 
-			M_ALIGN(m, len);
-	}
+	if (len < M_SIZE(m))
+		M_ALIGN(m, len);
 	m->m_len = len;
 	return (m);
 }
@@ -522,7 +436,7 @@ m_prepend(struct mbuf *m, int len, int how)
 /*
  * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
  * continuing for "len" bytes.  If len is M_COPYALL, copy to end of mbuf.
- * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller.
+ * The wait parameter is a choice of M_WAITOK/M_NOWAIT from caller.
  * Note that the copy is read-only, because clusters are not copied,
  * only their reference counts are incremented.
  */
@@ -547,17 +461,17 @@ m_copym(struct mbuf *m, int off0, int len, int wait)
 		m = m->m_next;
 	}
 	np = &top;
-	top = 0;
+	top = NULL;
 	while (len > 0) {
 		if (m == NULL) {
-			KASSERT(len == M_COPYALL, 
+			KASSERT(len == M_COPYALL,
 			    ("m_copym, length > size of mbuf chain"));
 			break;
 		}
 		if (copyhdr)
-			MGETHDR(n, wait, m->m_type);
+			n = m_gethdr(wait, m->m_type);
 		else
-			MGET(n, wait, m->m_type);
+			n = m_get(wait, m->m_type);
 		*np = n;
 		if (n == NULL)
 			goto nospace;
@@ -583,165 +497,14 @@ m_copym(struct mbuf *m, int off0, int len, int wait)
 		m = m->m_next;
 		np = &n->m_next;
 	}
-	if (top == NULL)
-		mbstat.m_mcfail++;	/* XXX: No consistency. */
 
 	return (top);
 nospace:
 	m_freem(top);
-	mbstat.m_mcfail++;	/* XXX: No consistency. */
 	return (NULL);
 }
 
 /*
- * Returns mbuf chain with new head for the prepending case.
- * Copies from mbuf (chain) n from off for len to mbuf (chain) m
- * either prepending or appending the data.
- * The resulting mbuf (chain) m is fully writeable.
- * m is destination (is made writeable)
- * n is source, off is offset in source, len is len from offset
- * dir, 0 append, 1 prepend
- * how, wait or nowait
- */
-
-static int
-m_bcopyxxx(void *s, void *t, u_int len)
-{
-	bcopy(s, t, (size_t)len);
-	return 0;
-}
-
-struct mbuf *
-m_copymdata(struct mbuf *m, struct mbuf *n, int off, int len,
-    int prep, int how)
-{
-	struct mbuf *mm, *x, *z, *prev = NULL;
-	caddr_t p;
-	int i, nlen = 0;
-	caddr_t buf[MLEN];
-
-	KASSERT(m != NULL && n != NULL, ("m_copymdata, no target or source"));
-	KASSERT(off >= 0, ("m_copymdata, negative off %d", off));
-	KASSERT(len >= 0, ("m_copymdata, negative len %d", len));
-	KASSERT(prep == 0 || prep == 1, ("m_copymdata, unknown direction %d", prep));
-
-	mm = m;
-	if (!prep) {
-		while(mm->m_next) {
-			prev = mm;
-			mm = mm->m_next;
-		}
-	}
-	for (z = n; z != NULL; z = z->m_next)
-		nlen += z->m_len;
-	if (len == M_COPYALL)
-		len = nlen - off;
-	if (off + len > nlen || len < 1)
-		return NULL;
-
-	if (!M_WRITABLE(mm)) {
-		/* XXX: Use proper m_xxx function instead. */
-		x = m_getcl(how, MT_DATA, mm->m_flags);
-		if (x == NULL)
-			return NULL;
-		bcopy(mm->m_ext.ext_buf, x->m_ext.ext_buf, x->m_ext.ext_size);
-		p = x->m_ext.ext_buf + (mm->m_data - mm->m_ext.ext_buf);
-		x->m_data = p;
-		mm->m_next = NULL;
-		if (mm != m)
-			prev->m_next = x;
-		m_free(mm);
-		mm = x;
-	}
-
-	/*
-	 * Append/prepend the data.  Allocating mbufs as necessary.
-	 */
-	/* Shortcut if enough free space in first/last mbuf. */
-	if (!prep && M_TRAILINGSPACE(mm) >= len) {
-		m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t) +
-			 mm->m_len);
-		mm->m_len += len;
-		mm->m_pkthdr.len += len;
-		return m;
-	}
-	if (prep && M_LEADINGSPACE(mm) >= len) {
-		mm->m_data = mtod(mm, caddr_t) - len;
-		m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t));
-		mm->m_len += len;
-		mm->m_pkthdr.len += len;
-		return mm;
-	}
-
-	/* Expand first/last mbuf to cluster if possible. */
-	if (!prep && !(mm->m_flags & M_EXT) && len > M_TRAILINGSPACE(mm)) {
-		bcopy(mm->m_data, &buf, mm->m_len);
-		m_clget(mm, how);
-		if (!(mm->m_flags & M_EXT))
-			return NULL;
-		bcopy(&buf, mm->m_ext.ext_buf, mm->m_len);
-		mm->m_data = mm->m_ext.ext_buf;
-		mm->m_pkthdr.header = NULL;
-	}
-	if (prep && !(mm->m_flags & M_EXT) && len > M_LEADINGSPACE(mm)) {
-		bcopy(mm->m_data, &buf, mm->m_len);
-		m_clget(mm, how);
-		if (!(mm->m_flags & M_EXT))
-			return NULL;
-		bcopy(&buf, (caddr_t *)mm->m_ext.ext_buf +
-		       mm->m_ext.ext_size - mm->m_len, mm->m_len);
-		mm->m_data = (caddr_t)mm->m_ext.ext_buf +
-			      mm->m_ext.ext_size - mm->m_len;
-		mm->m_pkthdr.header = NULL;
-	}
-
-	/* Append/prepend as many mbuf (clusters) as necessary to fit len. */
-	if (!prep && len > M_TRAILINGSPACE(mm)) {
-		if (!m_getm(mm, len - M_TRAILINGSPACE(mm), how, MT_DATA))
-			return NULL;
-	}
-	if (prep && len > M_LEADINGSPACE(mm)) {
-		if (!(z = m_getm(NULL, len - M_LEADINGSPACE(mm), how, MT_DATA)))
-			return NULL;
-		i = 0;
-		for (x = z; x != NULL; x = x->m_next) {
-			i += x->m_flags & M_EXT ? x->m_ext.ext_size :
-			      (x->m_flags & M_PKTHDR ? MHLEN : MLEN);
-			if (!x->m_next)
-				break;
-		}
-		z->m_data += i - len;
-		m_move_pkthdr(mm, z);
-		x->m_next = mm;
-		mm = z;
-	}
-
-	/* Seek to start position in source mbuf. Optimization for long chains. */
-	while (off > 0) {
-		if (off < n->m_len)
-			break;
-		off -= n->m_len;
-		n = n->m_next;
-	}
-
-	/* Copy data into target mbuf. */
-	z = mm;
-	while (len > 0) {
-		KASSERT(z != NULL, ("m_copymdata, falling off target edge"));
-		i = M_TRAILINGSPACE(z);
-		m_apply(n, off, i, m_bcopyxxx, mtod(z, caddr_t) + z->m_len);
-		z->m_len += i;
-		/* fixup pkthdr.len if necessary */
-		if ((prep ? mm : m)->m_flags & M_PKTHDR)
-			(prep ? mm : m)->m_pkthdr.len += i;
-		off += i;
-		len -= i;
-		z = z->m_next;
-	}
-	return (prep ? mm : m);
-}
-
-/*
  * Copy an entire packet, including header (which must be present).
  * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'.
  * Note that the copy is read-only, because clusters are not copied,
@@ -756,7 +519,7 @@ m_copypacket(struct mbuf *m, int how)
 	struct mbuf *top, *n, *o;
 
 	MBUF_CHECKSLEEP(how);
-	MGET(n, how, m->m_type);
+	n = m_get(how, m->m_type);
 	top = n;
 	if (n == NULL)
 		goto nospace;
@@ -774,7 +537,7 @@ m_copypacket(struct mbuf *m, int how)
 
 	m = m->m_next;
 	while (m) {
-		MGET(o, how, m->m_type);
+		o = m_get(how, m->m_type);
 		if (o == NULL)
 			goto nospace;
 
@@ -794,7 +557,6 @@ m_copypacket(struct mbuf *m, int how)
 	return top;
 nospace:
 	m_freem(top);
-	mbstat.m_mcfail++;	/* XXX: No consistency. */ 
 	return (NULL);
 }
 
@@ -833,7 +595,7 @@ m_copydata(const struct mbuf *m, int off, int len, caddr_t cp)
  * you need a writable copy of an mbuf chain.
  */
 struct mbuf *
-m_dup(struct mbuf *m, int how)
+m_dup(const struct mbuf *m, int how)
 {
 	struct mbuf **p, *top = NULL;
 	int remain, moff, nsize;
@@ -869,6 +631,7 @@ m_dup(struct mbuf *m, int how)
 			}
 			if ((n->m_flags & M_EXT) == 0)
 				nsize = MHLEN;
+			n->m_flags &= ~M_RDONLY;
 		}
 		n->m_len = 0;
 
@@ -898,7 +661,6 @@ m_dup(struct mbuf *m, int how)
 
 nospace:
 	m_freem(top);
-	mbstat.m_mcfail++;	/* XXX: No consistency. */
 	return (NULL);
 }
 
@@ -927,6 +689,22 @@ m_cat(struct mbuf *m, struct mbuf *n)
 	}
 }
 
+/*
+ * Concatenate two pkthdr mbuf chains.
+ */
+void
+m_catpkt(struct mbuf *m, struct mbuf *n)
+{
+
+	M_ASSERTPKTHDR(m);
+	M_ASSERTPKTHDR(n);
+
+	m->m_pkthdr.len += n->m_pkthdr.len;
+	m_demote(n, 1, 0);
+
+	m_cat(m, n);
+}
+
 void
 m_adj(struct mbuf *mp, int req_len)
 {
@@ -1002,8 +780,8 @@ m_adj(struct mbuf *mp, int req_len)
 
 /*
  * Rearange an mbuf chain so that len bytes are contiguous
- * and in the data area of an mbuf (so that mtod and dtom
- * will work for a structure of size len).  Returns the resulting
+ * and in the data area of an mbuf (so that mtod will work
+ * for a structure of size len).  Returns the resulting
  * mbuf chain on success, frees it and returns null on failure.
  * If there is room, it will add up to max_protohdr-len extra bytes to the
  * contiguous region in an attempt to avoid being called next time.
@@ -1030,12 +808,11 @@ m_pullup(struct mbuf *n, int len)
 	} else {
 		if (len > MHLEN)
 			goto bad;
-		MGET(m, M_DONTWAIT, n->m_type);
+		m = m_get(M_NOWAIT, n->m_type);
 		if (m == NULL)
 			goto bad;
-		m->m_len = 0;
 		if (n->m_flags & M_PKTHDR)
-			M_MOVE_PKTHDR(m, n);
+			m_move_pkthdr(m, n);
 	}
 	space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
 	do {
@@ -1059,7 +836,6 @@ m_pullup(struct mbuf *n, int len)
 	return (m);
 bad:
 	m_freem(n);
-	mbstat.m_mpfail++;	/* XXX: No consistency. */
 	return (NULL);
 }
 
@@ -1068,8 +844,6 @@ bad:
  * the amount of empty space before the data in the new mbuf to be specified
  * (in the event that the caller expects to prepend later).
  */
-int MSFail;
-
 struct mbuf *
 m_copyup(struct mbuf *n, int len, int dstoff)
 {
@@ -1078,12 +852,11 @@ m_copyup(struct mbuf *n, int len, int dstoff)
 
 	if (len > (MHLEN - dstoff))
 		goto bad;
-	MGET(m, M_DONTWAIT, n->m_type);
+	m = m_get(M_NOWAIT, n->m_type);
 	if (m == NULL)
 		goto bad;
-	m->m_len = 0;
 	if (n->m_flags & M_PKTHDR)
-		M_MOVE_PKTHDR(m, n);
+		m_move_pkthdr(m, n);
 	m->m_data += dstoff;
 	space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
 	do {
@@ -1107,7 +880,6 @@ m_copyup(struct mbuf *n, int len, int dstoff)
 	return (m);
  bad:
 	m_freem(n);
-	MSFail++;
 	return (NULL);
 }
 
@@ -1133,8 +905,18 @@ m_split(struct mbuf *m0, int len0, int wait)
 	if (m == NULL)
 		return (NULL);
 	remain = m->m_len - len;
-	if (m0->m_flags & M_PKTHDR) {
-		MGETHDR(n, wait, m0->m_type);
+	if (m0->m_flags & M_PKTHDR && remain == 0) {
+		n = m_gethdr(wait, m0->m_type);
+		if (n == NULL)
+			return (NULL);
+		n->m_next = m->m_next;
+		m->m_next = NULL;
+		n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
+		n->m_pkthdr.len = m0->m_pkthdr.len - len0;
+		m0->m_pkthdr.len = len0;
+		return (n);
+	} else if (m0->m_flags & M_PKTHDR) {
+		n = m_gethdr(wait, m0->m_type);
 		if (n == NULL)
 			return (NULL);
 		n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
@@ -1144,7 +926,7 @@ m_split(struct mbuf *m0, int len0, int wait)
 			goto extpacket;
 		if (remain > MHLEN) {
 			/* m can't be the lead packet */
-			MH_ALIGN(n, 0);
+			M_ALIGN(n, 0);
 			n->m_next = m_split(m, len, wait);
 			if (n->m_next == NULL) {
 				(void) m_free(n);
@@ -1154,13 +936,13 @@ m_split(struct mbuf *m0, int len0, int wait)
 				return (n);
 			}
 		} else
-			MH_ALIGN(n, remain);
+			M_ALIGN(n, remain);
 	} else if (remain == 0) {
 		n = m->m_next;
 		m->m_next = NULL;
 		return (n);
 	} else {
-		MGET(n, wait, m->m_type);
+		n = m_get(wait, m->m_type);
 		if (n == NULL)
 			return (NULL);
 		M_ALIGN(n, remain);
@@ -1197,10 +979,10 @@ m_devget(char *buf, int totlen, int off, struct ifnet *ifp,
 	while (totlen > 0) {
 		if (top == NULL) {	/* First one, must be PKTHDR */
 			if (totlen + off >= MINCLSIZE) {
-				m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
+				m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
 				len = MCLBYTES;
 			} else {
-				m = m_gethdr(M_DONTWAIT, MT_DATA);
+				m = m_gethdr(M_NOWAIT, MT_DATA);
 				len = MHLEN;
 
 				/* Place initial small packet/header at end of mbuf */
@@ -1215,10 +997,10 @@ m_devget(char *buf, int totlen, int off, struct ifnet *ifp,
 			m->m_pkthdr.len = totlen;
 		} else {
 			if (totlen + off >= MINCLSIZE) {
-				m = m_getcl(M_DONTWAIT, MT_DATA, 0);
+				m = m_getcl(M_NOWAIT, MT_DATA, 0);
 				len = MCLBYTES;
 			} else {
-				m = m_get(M_DONTWAIT, MT_DATA);
+				m = m_get(M_NOWAIT, MT_DATA);
 				len = MLEN;
 			}
 			if (m == NULL) {
@@ -1262,7 +1044,7 @@ m_copyback(struct mbuf *m0, int off, int len, c_caddr_t cp)
 		off -= mlen;
 		totlen += mlen;
 		if (m->m_next == NULL) {
-			n = m_get(M_DONTWAIT, m->m_type);
+			n = m_get(M_NOWAIT, m->m_type);
 			if (n == NULL)
 				goto out;
 			bzero(mtod(n, caddr_t), MLEN);
@@ -1286,7 +1068,7 @@ m_copyback(struct mbuf *m0, int off, int len, c_caddr_t cp)
 		if (len == 0)
 			break;
 		if (m->m_next == NULL) {
-			n = m_get(M_DONTWAIT, m->m_type);
+			n = m_get(M_NOWAIT, m->m_type);
 			if (n == NULL)
 				break;
 			n->m_len = min(MLEN, len);
@@ -1330,7 +1112,7 @@ m_append(struct mbuf *m0, int len, c_caddr_t cp)
 		 * Allocate a new mbuf; could check space
 		 * and allocate a cluster instead.
 		 */
-		n = m_get(M_DONTWAIT, m->m_type);
+		n = m_get(M_NOWAIT, m->m_type);
 		if (n == NULL)
 			break;
 		n->m_len = min(MLEN, remainder);
@@ -1498,7 +1280,7 @@ m_defrag(struct mbuf *m0, int how)
 			goto nospace;
 	}
 #endif
-	
+
 	if (m0->m_pkthdr.len > MHLEN)
 		m_final = m_getcl(how, MT_DATA, M_PKTHDR);
 	else
@@ -1664,7 +1446,7 @@ m_fragment(struct mbuf *m0, int how, int length)
 
 	if (!(m0->m_flags & M_PKTHDR))
 		return (m0);
-	
+
 	if ((length == 0) || (length < -2))
 		return (m0);
 
@@ -1806,25 +1588,6 @@ m_mbuftouio(struct uio *uio, struct mbuf *m, int len)
 }
 
 /*
- * Set the m_data pointer of a newly-allocated mbuf
- * to place an object of the specified size at the
- * end of the mbuf, longword aligned.
- */
-void
-m_align(struct mbuf *m, int len)
-{
-	int adjust;
-
-	if (m->m_flags & M_EXT)
-		adjust = m->m_ext.ext_size - len;
-	else if (m->m_flags & M_PKTHDR)
-		adjust = MHLEN - len;
-	else
-		adjust = MLEN - len;
-	m->m_data += adjust &~ (sizeof(long)-1);
-}
-
-/*
  * Create a writable copy of the mbuf chain.  While doing this
  * we compact the chain with a goal of producing a chain with
  * at most two mbufs.  The second mbuf in this chain is likely
@@ -1859,7 +1622,7 @@ m_unshare(struct mbuf *m0, int how)
 			    m->m_len <= M_TRAILINGSPACE(mprev)) {
 				/* XXX: this ignores mbuf types */
 				memcpy(mtod(mprev, caddr_t) + mprev->m_len,
-				       mtod(m, caddr_t), m->m_len);
+				    mtod(m, caddr_t), m->m_len);
 				mprev->m_len += m->m_len;
 				mprev->m_next = m->m_next;	/* unlink from chain */
 				m_free(m);			/* reclaim mbuf */
@@ -1891,7 +1654,7 @@ m_unshare(struct mbuf *m0, int how)
 		    m->m_len <= M_TRAILINGSPACE(mprev)) {
 			/* XXX: this ignores mbuf types */
 			memcpy(mtod(mprev, caddr_t) + mprev->m_len,
-			       mtod(m, caddr_t), m->m_len);
+			    mtod(m, caddr_t), m->m_len);
 			mprev->m_len += m->m_len;
 			mprev->m_next = m->m_next;	/* unlink from chain */
 			m_free(m);			/* reclaim mbuf */
@@ -1902,43 +1665,23 @@ m_unshare(struct mbuf *m0, int how)
 		}
 
 		/*
-		 * Allocate new space to hold the copy...
-		 */
-		/* XXX why can M_PKTHDR be set past the first mbuf? */
-		if (mprev == NULL && (m->m_flags & M_PKTHDR)) {
-			/*
-			 * NB: if a packet header is present we must
-			 * allocate the mbuf separately from any cluster
-			 * because M_MOVE_PKTHDR will smash the data
-			 * pointer and drop the M_EXT marker.
-			 */
-			MGETHDR(n, how, m->m_type);
-			if (n == NULL) {
-				m_freem(m0);
-				return (NULL);
-			}
-			M_MOVE_PKTHDR(n, m);
-			MCLGET(n, how);
-			if ((n->m_flags & M_EXT) == 0) {
-				m_free(n);
-				m_freem(m0);
-				return (NULL);
-			}
-		} else {
-			n = m_getcl(how, m->m_type, m->m_flags);
-			if (n == NULL) {
-				m_freem(m0);
-				return (NULL);
-			}
-		}
-		/*
-		 * ... and copy the data.  We deal with jumbo mbufs
-		 * (i.e. m_len > MCLBYTES) by splitting them into
-		 * clusters.  We could just malloc a buffer and make
-		 * it external but too many device drivers don't know
-		 * how to break up the non-contiguous memory when
+		 * Allocate new space to hold the copy and copy the data.
+		 * We deal with jumbo mbufs (i.e. m_len > MCLBYTES) by
+		 * splitting them into clusters.  We could just malloc a
+		 * buffer and make it external but too many device drivers
+		 * don't know how to break up the non-contiguous memory when
 		 * doing DMA.
 		 */
+		n = m_getcl(how, m->m_type, m->m_flags & M_COPYFLAGS);
+		if (n == NULL) {
+			m_freem(m0);
+			return (NULL);
+		}
+		if (m->m_flags & M_PKTHDR) {
+			KASSERT(mprev == NULL, ("%s: m0 %p, m %p has M_PKTHDR",
+			    __func__, m0, m));
+			m_move_pkthdr(n, m);
+		}
 		len = m->m_len;
 		off = 0;
 		mfirst = n;
@@ -1949,7 +1692,7 @@ m_unshare(struct mbuf *m0, int how)
 			n->m_len = cc;
 			if (mlast != NULL)
 				mlast->m_next = n;
-			mlast = n;	
+			mlast = n;
 #if 0
 			newipsecstat.ips_clcopied++;
 #endif
@@ -1959,14 +1702,14 @@ m_unshare(struct mbuf *m0, int how)
 				break;
 			off += cc;
 
-			n = m_getcl(how, m->m_type, m->m_flags);
+			n = m_getcl(how, m->m_type, m->m_flags & M_COPYFLAGS);
 			if (n == NULL) {
 				m_freem(mfirst);
 				m_freem(m0);
 				return (NULL);
 			}
 		}
-		n->m_next = m->m_next; 
+		n->m_next = m->m_next;
 		if (mprev == NULL)
 			m0 = mfirst;		/* new head of chain */
 		else
@@ -2002,7 +1745,7 @@ m_profile(struct mbuf *m)
 	int segments = 0;
 	int used = 0;
 	int wasted = 0;
-	
+
 	while (m) {
 		segments++;
 		used += m->m_len;
@@ -2037,11 +1780,10 @@ mbprof_textify(void)
 	int offset;
 	char *c;
 	uint64_t *p;
-	
 
 	p = &mbprof.wasted[0];
 	c = mbprofbuf;
-	offset = snprintf(c, MP_MAXLINE + 10, 
+	offset = snprintf(c, MP_MAXLINE + 10,
 	    "wasted:\n"
 	    "%ju %ju %ju %ju %ju %ju %ju %ju "
 	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
@@ -2050,7 +1792,7 @@ mbprof_textify(void)
 #ifdef BIG_ARRAY
 	p = &mbprof.wasted[16];
 	c += offset;
-	offset = snprintf(c, MP_MAXLINE, 
+	offset = snprintf(c, MP_MAXLINE,
 	    "%ju %ju %ju %ju %ju %ju %ju %ju "
 	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
 	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
@@ -2058,7 +1800,7 @@ mbprof_textify(void)
 #endif
 	p = &mbprof.used[0];
 	c += offset;
-	offset = snprintf(c, MP_MAXLINE + 10, 
+	offset = snprintf(c, MP_MAXLINE + 10,
 	    "used:\n"
 	    "%ju %ju %ju %ju %ju %ju %ju %ju "
 	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
@@ -2067,7 +1809,7 @@ mbprof_textify(void)
 #ifdef BIG_ARRAY
 	p = &mbprof.used[16];
 	c += offset;
-	offset = snprintf(c, MP_MAXLINE, 
+	offset = snprintf(c, MP_MAXLINE,
 	    "%ju %ju %ju %ju %ju %ju %ju %ju "
 	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
 	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
@@ -2075,7 +1817,7 @@ mbprof_textify(void)
 #endif
 	p = &mbprof.segments[0];
 	c += offset;
-	offset = snprintf(c, MP_MAXLINE + 10, 
+	offset = snprintf(c, MP_MAXLINE + 10,
 	    "segments:\n"
 	    "%ju %ju %ju %ju %ju %ju %ju %ju "
 	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
@@ -2084,7 +1826,7 @@ mbprof_textify(void)
 #ifdef BIG_ARRAY
 	p = &mbprof.segments[16];
 	c += offset;
-	offset = snprintf(c, MP_MAXLINE, 
+	offset = snprintf(c, MP_MAXLINE,
 	    "%ju %ju %ju %ju %ju %ju %ju %ju "
 	    "%ju %ju %ju %ju %ju %ju %ju %jju",
 	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
@@ -2106,16 +1848,16 @@ static int
 mbprof_clr_handler(SYSCTL_HANDLER_ARGS)
 {
 	int clear, error;
- 
+
 	clear = 0;
 	error = sysctl_handle_int(oidp, &clear, 0, req);
 	if (error || !req->newptr)
 		return (error);
- 
+
 	if (clear) {
 		bzero(&mbprof, sizeof(mbprof));
 	}
- 
+
 	return (error);
 }