From c40e45b75eb76d79a05c7fa85c1fa9b5c728a12f Mon Sep 17 00:00:00 2001 From: Sebastian Huber Date: Fri, 7 Oct 2016 15:10:20 +0200 Subject: Update to FreeBSD head 2016-08-23 Git mirror commit 9fe7c416e6abb28b1398fd3e5687099846800cfd. --- freebsd/sys/kern/uipc_mbuf.c | 728 ++++++++++++++----------------------------- 1 file changed, 235 insertions(+), 493 deletions(-) (limited to 'freebsd/sys/kern/uipc_mbuf.c') diff --git a/freebsd/sys/kern/uipc_mbuf.c b/freebsd/sys/kern/uipc_mbuf.c index abf4dd3e..db4975ca 100644 --- a/freebsd/sys/kern/uipc_mbuf.c +++ b/freebsd/sys/kern/uipc_mbuf.c @@ -49,6 +49,51 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include + +SDT_PROBE_DEFINE5_XLATE(sdt, , , m__init, + "struct mbuf *", "mbufinfo_t *", + "uint32_t", "uint32_t", + "uint16_t", "uint16_t", + "uint32_t", "uint32_t", + "uint32_t", "uint32_t"); + +SDT_PROBE_DEFINE3_XLATE(sdt, , , m__gethdr, + "uint32_t", "uint32_t", + "uint16_t", "uint16_t", + "struct mbuf *", "mbufinfo_t *"); + +SDT_PROBE_DEFINE3_XLATE(sdt, , , m__get, + "uint32_t", "uint32_t", + "uint16_t", "uint16_t", + "struct mbuf *", "mbufinfo_t *"); + +SDT_PROBE_DEFINE4_XLATE(sdt, , , m__getcl, + "uint32_t", "uint32_t", + "uint16_t", "uint16_t", + "uint32_t", "uint32_t", + "struct mbuf *", "mbufinfo_t *"); + +SDT_PROBE_DEFINE3_XLATE(sdt, , , m__clget, + "struct mbuf *", "mbufinfo_t *", + "uint32_t", "uint32_t", + "uint32_t", "uint32_t"); + +SDT_PROBE_DEFINE4_XLATE(sdt, , , m__cljget, + "struct mbuf *", "mbufinfo_t *", + "uint32_t", "uint32_t", + "uint32_t", "uint32_t", + "void*", "void*"); + +SDT_PROBE_DEFINE(sdt, , , m__cljset); + +SDT_PROBE_DEFINE1_XLATE(sdt, , , m__free, + "struct mbuf *", "mbufinfo_t *"); + +SDT_PROBE_DEFINE1_XLATE(sdt, , , m__freem, + "struct mbuf *", "mbufinfo_t *"); + +#include int max_linkhdr; int max_protohdr; @@ -87,224 +132,93 @@ SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragrandomfailures, CTLFLAG_RW, #endif /* - * Allocate a given length worth of mbufs and/or clusters (whatever fits - * best) and return a pointer to the top of the allocated chain. If an - * existing mbuf chain is provided, then we will append the new chain - * to the existing one but still return the top of the newly allocated - * chain. + * Ensure the correct size of various mbuf parameters. It could be off due + * to compiler-induced padding and alignment artifacts. */ -struct mbuf * -m_getm2(struct mbuf *m, int len, int how, short type, int flags) -{ - struct mbuf *mb, *nm = NULL, *mtail = NULL; - - KASSERT(len >= 0, ("%s: len is < 0", __func__)); - - /* Validate flags. */ - flags &= (M_PKTHDR | M_EOR); - - /* Packet header mbuf must be first in chain. */ - if ((flags & M_PKTHDR) && m != NULL) - flags &= ~M_PKTHDR; - - /* Loop and append maximum sized mbufs to the chain tail. */ - while (len > 0) { - if (len > MCLBYTES) - mb = m_getjcl(how, type, (flags & M_PKTHDR), - MJUMPAGESIZE); - else if (len >= MINCLSIZE) - mb = m_getcl(how, type, (flags & M_PKTHDR)); - else if (flags & M_PKTHDR) - mb = m_gethdr(how, type); - else - mb = m_get(how, type); - - /* Fail the whole operation if one mbuf can't be allocated. */ - if (mb == NULL) { - if (nm != NULL) - m_freem(nm); - return (NULL); - } - - /* Book keeping. */ - len -= (mb->m_flags & M_EXT) ? mb->m_ext.ext_size : - ((mb->m_flags & M_PKTHDR) ? MHLEN : MLEN); - if (mtail != NULL) - mtail->m_next = mb; - else - nm = mb; - mtail = mb; - flags &= ~M_PKTHDR; /* Only valid on the first mbuf. */ - } - if (flags & M_EOR) - mtail->m_flags |= M_EOR; /* Only valid on the last mbuf. */ - - /* If mbuf was supplied, append new chain to the end of it. */ - if (m != NULL) { - for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next) - ; - mtail->m_next = nm; - mtail->m_flags &= ~M_EOR; - } else - m = nm; - - return (m); -} +CTASSERT(MSIZE - offsetof(struct mbuf, m_dat) == MLEN); +CTASSERT(MSIZE - offsetof(struct mbuf, m_pktdat) == MHLEN); /* - * Free an entire chain of mbufs and associated external buffers, if - * applicable. + * mbuf data storage should be 64-bit aligned regardless of architectural + * pointer size; check this is the case with and without a packet header. */ -void -m_freem(struct mbuf *mb) -{ +CTASSERT(offsetof(struct mbuf, m_dat) % 8 == 0); +CTASSERT(offsetof(struct mbuf, m_pktdat) % 8 == 0); - while (mb != NULL) - mb = m_free(mb); -} - -/*- - * Configure a provided mbuf to refer to the provided external storage - * buffer and setup a reference count for said buffer. If the setting - * up of the reference count fails, the M_EXT bit will not be set. If - * successfull, the M_EXT bit is set in the mbuf's flags. - * - * Arguments: - * mb The existing mbuf to which to attach the provided buffer. - * buf The address of the provided external storage buffer. - * size The size of the provided buffer. - * freef A pointer to a routine that is responsible for freeing the - * provided external storage buffer. - * args A pointer to an argument structure (of any type) to be passed - * to the provided freef routine (may be NULL). - * flags Any other flags to be passed to the provided mbuf. - * type The type that the external storage buffer should be - * labeled with. +/* + * While the specific values here don't matter too much (i.e., +/- a few + * words), we do want to ensure that changes to these values are carefully + * reasoned about and properly documented. This is especially the case as + * network-protocol and device-driver modules encode these layouts, and must + * be recompiled if the structures change. Check these values at compile time + * against the ones documented in comments in mbuf.h. * - * Returns: - * Nothing. + * NB: Possibly they should be documented there via #define's and not just + * comments. */ -void -m_extadd(struct mbuf *mb, caddr_t buf, u_int size, - void (*freef)(void *, void *), void *arg1, void *arg2, int flags, int type) -{ - KASSERT(type != EXT_CLUSTER, ("%s: EXT_CLUSTER not allowed", __func__)); - - if (type != EXT_EXTREF) - mb->m_ext.ref_cnt = (u_int *)uma_zalloc(zone_ext_refcnt, M_NOWAIT); - if (mb->m_ext.ref_cnt != NULL) { - *(mb->m_ext.ref_cnt) = 1; - mb->m_flags |= (M_EXT | flags); - mb->m_ext.ext_buf = buf; - mb->m_data = mb->m_ext.ext_buf; - mb->m_ext.ext_size = size; - mb->m_ext.ext_free = freef; - mb->m_ext.ext_arg1 = arg1; - mb->m_ext.ext_arg2 = arg2; - mb->m_ext.ext_type = type; - } -} +#if defined(__LP64__) +CTASSERT(offsetof(struct mbuf, m_dat) == 32); +CTASSERT(sizeof(struct pkthdr) == 56); +CTASSERT(sizeof(struct m_ext) == 48); +#else +CTASSERT(offsetof(struct mbuf, m_dat) == 24); +CTASSERT(sizeof(struct pkthdr) == 48); +CTASSERT(sizeof(struct m_ext) == 28); +#endif /* - * Non-directly-exported function to clean up after mbufs with M_EXT - * storage attached to them if the reference count hits 1. + * Assert that the queue(3) macros produce code of the same size as an old + * plain pointer does. */ -void -mb_free_ext(struct mbuf *m) -{ - int skipmbuf; - - KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__)); - KASSERT(m->m_ext.ref_cnt != NULL, ("%s: ref_cnt not set", __func__)); - - - /* - * check if the header is embedded in the cluster - */ - skipmbuf = (m->m_flags & M_NOFREE); - - /* Free attached storage if this mbuf is the only reference to it. */ - if (*(m->m_ext.ref_cnt) == 1 || - atomic_fetchadd_int(m->m_ext.ref_cnt, -1) == 1) { - switch (m->m_ext.ext_type) { - case EXT_PACKET: /* The packet zone is special. */ - if (*(m->m_ext.ref_cnt) == 0) - *(m->m_ext.ref_cnt) = 1; - uma_zfree(zone_pack, m); - return; /* Job done. */ - case EXT_CLUSTER: - uma_zfree(zone_clust, m->m_ext.ext_buf); - break; - case EXT_JUMBOP: - uma_zfree(zone_jumbop, m->m_ext.ext_buf); - break; - case EXT_JUMBO9: - uma_zfree(zone_jumbo9, m->m_ext.ext_buf); - break; - case EXT_JUMBO16: - uma_zfree(zone_jumbo16, m->m_ext.ext_buf); - break; - case EXT_SFBUF: - case EXT_NET_DRV: - case EXT_MOD_TYPE: - case EXT_DISPOSABLE: - *(m->m_ext.ref_cnt) = 0; - uma_zfree(zone_ext_refcnt, __DEVOLATILE(u_int *, - m->m_ext.ref_cnt)); - /* FALLTHROUGH */ - case EXT_EXTREF: - KASSERT(m->m_ext.ext_free != NULL, - ("%s: ext_free not set", __func__)); - (*(m->m_ext.ext_free))(m->m_ext.ext_arg1, - m->m_ext.ext_arg2); - break; - default: - KASSERT(m->m_ext.ext_type == 0, - ("%s: unknown ext_type", __func__)); - } - } - if (skipmbuf) - return; - - /* - * Free this mbuf back to the mbuf zone with all m_ext - * information purged. - */ - m->m_ext.ext_buf = NULL; - m->m_ext.ext_free = NULL; - m->m_ext.ext_arg1 = NULL; - m->m_ext.ext_arg2 = NULL; - m->m_ext.ref_cnt = NULL; - m->m_ext.ext_size = 0; - m->m_ext.ext_type = 0; - m->m_flags &= ~M_EXT; - uma_zfree(zone_mbuf, m); -} +#ifdef INVARIANTS +static struct mbuf m_assertbuf; +CTASSERT(sizeof(m_assertbuf.m_slist) == sizeof(m_assertbuf.m_next)); +CTASSERT(sizeof(m_assertbuf.m_stailq) == sizeof(m_assertbuf.m_next)); +CTASSERT(sizeof(m_assertbuf.m_slistpkt) == sizeof(m_assertbuf.m_nextpkt)); +CTASSERT(sizeof(m_assertbuf.m_stailqpkt) == sizeof(m_assertbuf.m_nextpkt)); +#endif /* * Attach the cluster from *m to *n, set up m_ext in *n * and bump the refcount of the cluster. */ -static void +void mb_dupcl(struct mbuf *n, struct mbuf *m) { - KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__)); - KASSERT(m->m_ext.ref_cnt != NULL, ("%s: ref_cnt not set", __func__)); - KASSERT((n->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__)); + volatile u_int *refcnt; - if (*(m->m_ext.ref_cnt) == 1) - *(m->m_ext.ref_cnt) += 1; - else - atomic_add_int(m->m_ext.ref_cnt, 1); - n->m_ext.ext_buf = m->m_ext.ext_buf; - n->m_ext.ext_free = m->m_ext.ext_free; - n->m_ext.ext_arg1 = m->m_ext.ext_arg1; - n->m_ext.ext_arg2 = m->m_ext.ext_arg2; - n->m_ext.ext_size = m->m_ext.ext_size; - n->m_ext.ref_cnt = m->m_ext.ref_cnt; - n->m_ext.ext_type = m->m_ext.ext_type; + KASSERT(m->m_flags & M_EXT, ("%s: M_EXT not set on %p", __func__, m)); + KASSERT(!(n->m_flags & M_EXT), ("%s: M_EXT set on %p", __func__, n)); + + n->m_ext = m->m_ext; n->m_flags |= M_EXT; n->m_flags |= m->m_flags & M_RDONLY; + + /* See if this is the mbuf that holds the embedded refcount. */ + if (m->m_ext.ext_flags & EXT_FLAG_EMBREF) { + refcnt = n->m_ext.ext_cnt = &m->m_ext.ext_count; + n->m_ext.ext_flags &= ~EXT_FLAG_EMBREF; + } else { + KASSERT(m->m_ext.ext_cnt != NULL, + ("%s: no refcounting pointer on %p", __func__, m)); + refcnt = m->m_ext.ext_cnt; + } + + if (*refcnt == 1) + *refcnt += 1; + else + atomic_add_int(refcnt, 1); +} + +void +m_demote_pkthdr(struct mbuf *m) +{ + + M_ASSERTPKTHDR(m); + + m_tag_delete_chain(m, NULL); + m->m_flags &= ~M_PKTHDR; + bzero(&m->m_pkthdr, sizeof(struct pkthdr)); } /* @@ -313,23 +227,16 @@ mb_dupcl(struct mbuf *n, struct mbuf *m) * cleaned too. */ void -m_demote(struct mbuf *m0, int all) +m_demote(struct mbuf *m0, int all, int flags) { struct mbuf *m; for (m = all ? m0 : m0->m_next; m != NULL; m = m->m_next) { - if (m->m_flags & M_PKTHDR) { - m_tag_delete_chain(m, NULL); - m->m_flags &= ~M_PKTHDR; - bzero(&m->m_pkthdr, sizeof(struct pkthdr)); - } - if (m != m0 && m->m_nextpkt != NULL) { - KASSERT(m->m_nextpkt == NULL, - ("%s: m_nextpkt not NULL", __func__)); - m_freem(m->m_nextpkt); - m->m_nextpkt = NULL; - } - m->m_flags = m->m_flags & (M_EXT|M_RDONLY|M_FREELIST|M_NOFREE); + KASSERT(m->m_nextpkt == NULL, ("%s: m_nextpkt in m %p, m0 %p", + __func__, m, m0)); + if (m->m_flags & M_PKTHDR) + m_demote_pkthdr(m); + m->m_flags = m->m_flags & (M_EXT | M_RDONLY | M_NOFREE | flags); } } @@ -349,7 +256,7 @@ m_sanity(struct mbuf *m0, int sanitize) #ifdef INVARIANTS #define M_SANITY_ACTION(s) panic("mbuf %p: " s, m) -#else +#else #define M_SANITY_ACTION(s) printf("mbuf %p: " s, m) #endif @@ -359,22 +266,14 @@ m_sanity(struct mbuf *m0, int sanitize) * unrelated kernel memory before or after us is trashed. * No way to recover from that. */ - a = ((m->m_flags & M_EXT) ? m->m_ext.ext_buf : - ((m->m_flags & M_PKTHDR) ? (caddr_t)(&m->m_pktdat) : - (caddr_t)(&m->m_dat)) ); - b = (caddr_t)(a + (m->m_flags & M_EXT ? m->m_ext.ext_size : - ((m->m_flags & M_PKTHDR) ? MHLEN : MLEN))); + a = M_START(m); + b = a + M_SIZE(m); if ((caddr_t)m->m_data < a) M_SANITY_ACTION("m_data outside mbuf data range left"); if ((caddr_t)m->m_data > b) M_SANITY_ACTION("m_data outside mbuf data range right"); if ((caddr_t)m->m_data + m->m_len > b) M_SANITY_ACTION("m_data + m_len exeeds mbuf space"); - if ((m->m_flags & M_PKTHDR) && m->m_pkthdr.header) { - if ((caddr_t)m->m_pkthdr.header < a || - (caddr_t)m->m_pkthdr.header > b) - M_SANITY_ACTION("m_pkthdr.header outside mbuf data range"); - } /* m->m_nextpkt may only be set on first mbuf in chain. */ if (m != m0 && m->m_nextpkt != NULL) { @@ -421,6 +320,26 @@ m_sanity(struct mbuf *m0, int sanitize) #undef M_SANITY_ACTION } +/* + * Non-inlined part of m_init(). + */ +int +m_pkthdr_init(struct mbuf *m, int how) +{ +#ifdef MAC + int error; +#endif + m->m_data = m->m_pktdat; + bzero(&m->m_pkthdr, sizeof(m->m_pkthdr)); +#ifdef MAC + /* If the label init fails, fail the alloc */ + error = mac_mbuf_init(m, how); + if (error) + return (error); +#endif + + return (0); +} /* * "Move" mbuf pkthdr from "from" to "to". @@ -458,14 +377,14 @@ m_move_pkthdr(struct mbuf *to, struct mbuf *from) * In particular, this does a deep copy of the packet tags. */ int -m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how) +m_dup_pkthdr(struct mbuf *to, const struct mbuf *from, int how) { #if 0 /* * The mbuf allocator only initializes the pkthdr - * when the mbuf is allocated with MGETHDR. Many users - * (e.g. m_copy*, m_prepend) use MGET and then + * when the mbuf is allocated with m_gethdr(). Many users + * (e.g. m_copy*, m_prepend) use m_get() and then * smash the pkthdr as needed causing these * assertions to trip. For now just disable them. */ @@ -483,7 +402,7 @@ m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how) to->m_data = to->m_pktdat; to->m_pkthdr = from->m_pkthdr; SLIST_INIT(&to->m_pkthdr.tags); - return (m_tag_copy_chain(to, from, MBTOM(how))); + return (m_tag_copy_chain(to, from, how)); } /* @@ -497,24 +416,19 @@ m_prepend(struct mbuf *m, int len, int how) struct mbuf *mn; if (m->m_flags & M_PKTHDR) - MGETHDR(mn, how, m->m_type); + mn = m_gethdr(how, m->m_type); else - MGET(mn, how, m->m_type); + mn = m_get(how, m->m_type); if (mn == NULL) { m_freem(m); return (NULL); } if (m->m_flags & M_PKTHDR) - M_MOVE_PKTHDR(mn, m); + m_move_pkthdr(mn, m); mn->m_next = m; m = mn; - if(m->m_flags & M_PKTHDR) { - if (len < MHLEN) - MH_ALIGN(m, len); - } else { - if (len < MLEN) - M_ALIGN(m, len); - } + if (len < M_SIZE(m)) + M_ALIGN(m, len); m->m_len = len; return (m); } @@ -522,7 +436,7 @@ m_prepend(struct mbuf *m, int len, int how) /* * Make a copy of an mbuf chain starting "off0" bytes from the beginning, * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. - * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller. + * The wait parameter is a choice of M_WAITOK/M_NOWAIT from caller. * Note that the copy is read-only, because clusters are not copied, * only their reference counts are incremented. */ @@ -547,17 +461,17 @@ m_copym(struct mbuf *m, int off0, int len, int wait) m = m->m_next; } np = ⊤ - top = 0; + top = NULL; while (len > 0) { if (m == NULL) { - KASSERT(len == M_COPYALL, + KASSERT(len == M_COPYALL, ("m_copym, length > size of mbuf chain")); break; } if (copyhdr) - MGETHDR(n, wait, m->m_type); + n = m_gethdr(wait, m->m_type); else - MGET(n, wait, m->m_type); + n = m_get(wait, m->m_type); *np = n; if (n == NULL) goto nospace; @@ -583,164 +497,13 @@ m_copym(struct mbuf *m, int off0, int len, int wait) m = m->m_next; np = &n->m_next; } - if (top == NULL) - mbstat.m_mcfail++; /* XXX: No consistency. */ return (top); nospace: m_freem(top); - mbstat.m_mcfail++; /* XXX: No consistency. */ return (NULL); } -/* - * Returns mbuf chain with new head for the prepending case. - * Copies from mbuf (chain) n from off for len to mbuf (chain) m - * either prepending or appending the data. - * The resulting mbuf (chain) m is fully writeable. - * m is destination (is made writeable) - * n is source, off is offset in source, len is len from offset - * dir, 0 append, 1 prepend - * how, wait or nowait - */ - -static int -m_bcopyxxx(void *s, void *t, u_int len) -{ - bcopy(s, t, (size_t)len); - return 0; -} - -struct mbuf * -m_copymdata(struct mbuf *m, struct mbuf *n, int off, int len, - int prep, int how) -{ - struct mbuf *mm, *x, *z, *prev = NULL; - caddr_t p; - int i, nlen = 0; - caddr_t buf[MLEN]; - - KASSERT(m != NULL && n != NULL, ("m_copymdata, no target or source")); - KASSERT(off >= 0, ("m_copymdata, negative off %d", off)); - KASSERT(len >= 0, ("m_copymdata, negative len %d", len)); - KASSERT(prep == 0 || prep == 1, ("m_copymdata, unknown direction %d", prep)); - - mm = m; - if (!prep) { - while(mm->m_next) { - prev = mm; - mm = mm->m_next; - } - } - for (z = n; z != NULL; z = z->m_next) - nlen += z->m_len; - if (len == M_COPYALL) - len = nlen - off; - if (off + len > nlen || len < 1) - return NULL; - - if (!M_WRITABLE(mm)) { - /* XXX: Use proper m_xxx function instead. */ - x = m_getcl(how, MT_DATA, mm->m_flags); - if (x == NULL) - return NULL; - bcopy(mm->m_ext.ext_buf, x->m_ext.ext_buf, x->m_ext.ext_size); - p = x->m_ext.ext_buf + (mm->m_data - mm->m_ext.ext_buf); - x->m_data = p; - mm->m_next = NULL; - if (mm != m) - prev->m_next = x; - m_free(mm); - mm = x; - } - - /* - * Append/prepend the data. Allocating mbufs as necessary. - */ - /* Shortcut if enough free space in first/last mbuf. */ - if (!prep && M_TRAILINGSPACE(mm) >= len) { - m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t) + - mm->m_len); - mm->m_len += len; - mm->m_pkthdr.len += len; - return m; - } - if (prep && M_LEADINGSPACE(mm) >= len) { - mm->m_data = mtod(mm, caddr_t) - len; - m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t)); - mm->m_len += len; - mm->m_pkthdr.len += len; - return mm; - } - - /* Expand first/last mbuf to cluster if possible. */ - if (!prep && !(mm->m_flags & M_EXT) && len > M_TRAILINGSPACE(mm)) { - bcopy(mm->m_data, &buf, mm->m_len); - m_clget(mm, how); - if (!(mm->m_flags & M_EXT)) - return NULL; - bcopy(&buf, mm->m_ext.ext_buf, mm->m_len); - mm->m_data = mm->m_ext.ext_buf; - mm->m_pkthdr.header = NULL; - } - if (prep && !(mm->m_flags & M_EXT) && len > M_LEADINGSPACE(mm)) { - bcopy(mm->m_data, &buf, mm->m_len); - m_clget(mm, how); - if (!(mm->m_flags & M_EXT)) - return NULL; - bcopy(&buf, (caddr_t *)mm->m_ext.ext_buf + - mm->m_ext.ext_size - mm->m_len, mm->m_len); - mm->m_data = (caddr_t)mm->m_ext.ext_buf + - mm->m_ext.ext_size - mm->m_len; - mm->m_pkthdr.header = NULL; - } - - /* Append/prepend as many mbuf (clusters) as necessary to fit len. */ - if (!prep && len > M_TRAILINGSPACE(mm)) { - if (!m_getm(mm, len - M_TRAILINGSPACE(mm), how, MT_DATA)) - return NULL; - } - if (prep && len > M_LEADINGSPACE(mm)) { - if (!(z = m_getm(NULL, len - M_LEADINGSPACE(mm), how, MT_DATA))) - return NULL; - i = 0; - for (x = z; x != NULL; x = x->m_next) { - i += x->m_flags & M_EXT ? x->m_ext.ext_size : - (x->m_flags & M_PKTHDR ? MHLEN : MLEN); - if (!x->m_next) - break; - } - z->m_data += i - len; - m_move_pkthdr(mm, z); - x->m_next = mm; - mm = z; - } - - /* Seek to start position in source mbuf. Optimization for long chains. */ - while (off > 0) { - if (off < n->m_len) - break; - off -= n->m_len; - n = n->m_next; - } - - /* Copy data into target mbuf. */ - z = mm; - while (len > 0) { - KASSERT(z != NULL, ("m_copymdata, falling off target edge")); - i = M_TRAILINGSPACE(z); - m_apply(n, off, i, m_bcopyxxx, mtod(z, caddr_t) + z->m_len); - z->m_len += i; - /* fixup pkthdr.len if necessary */ - if ((prep ? mm : m)->m_flags & M_PKTHDR) - (prep ? mm : m)->m_pkthdr.len += i; - off += i; - len -= i; - z = z->m_next; - } - return (prep ? mm : m); -} - /* * Copy an entire packet, including header (which must be present). * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'. @@ -756,7 +519,7 @@ m_copypacket(struct mbuf *m, int how) struct mbuf *top, *n, *o; MBUF_CHECKSLEEP(how); - MGET(n, how, m->m_type); + n = m_get(how, m->m_type); top = n; if (n == NULL) goto nospace; @@ -774,7 +537,7 @@ m_copypacket(struct mbuf *m, int how) m = m->m_next; while (m) { - MGET(o, how, m->m_type); + o = m_get(how, m->m_type); if (o == NULL) goto nospace; @@ -794,7 +557,6 @@ m_copypacket(struct mbuf *m, int how) return top; nospace: m_freem(top); - mbstat.m_mcfail++; /* XXX: No consistency. */ return (NULL); } @@ -833,7 +595,7 @@ m_copydata(const struct mbuf *m, int off, int len, caddr_t cp) * you need a writable copy of an mbuf chain. */ struct mbuf * -m_dup(struct mbuf *m, int how) +m_dup(const struct mbuf *m, int how) { struct mbuf **p, *top = NULL; int remain, moff, nsize; @@ -869,6 +631,7 @@ m_dup(struct mbuf *m, int how) } if ((n->m_flags & M_EXT) == 0) nsize = MHLEN; + n->m_flags &= ~M_RDONLY; } n->m_len = 0; @@ -898,7 +661,6 @@ m_dup(struct mbuf *m, int how) nospace: m_freem(top); - mbstat.m_mcfail++; /* XXX: No consistency. */ return (NULL); } @@ -927,6 +689,22 @@ m_cat(struct mbuf *m, struct mbuf *n) } } +/* + * Concatenate two pkthdr mbuf chains. + */ +void +m_catpkt(struct mbuf *m, struct mbuf *n) +{ + + M_ASSERTPKTHDR(m); + M_ASSERTPKTHDR(n); + + m->m_pkthdr.len += n->m_pkthdr.len; + m_demote(n, 1, 0); + + m_cat(m, n); +} + void m_adj(struct mbuf *mp, int req_len) { @@ -1002,8 +780,8 @@ m_adj(struct mbuf *mp, int req_len) /* * Rearange an mbuf chain so that len bytes are contiguous - * and in the data area of an mbuf (so that mtod and dtom - * will work for a structure of size len). Returns the resulting + * and in the data area of an mbuf (so that mtod will work + * for a structure of size len). Returns the resulting * mbuf chain on success, frees it and returns null on failure. * If there is room, it will add up to max_protohdr-len extra bytes to the * contiguous region in an attempt to avoid being called next time. @@ -1030,12 +808,11 @@ m_pullup(struct mbuf *n, int len) } else { if (len > MHLEN) goto bad; - MGET(m, M_DONTWAIT, n->m_type); + m = m_get(M_NOWAIT, n->m_type); if (m == NULL) goto bad; - m->m_len = 0; if (n->m_flags & M_PKTHDR) - M_MOVE_PKTHDR(m, n); + m_move_pkthdr(m, n); } space = &m->m_dat[MLEN] - (m->m_data + m->m_len); do { @@ -1059,7 +836,6 @@ m_pullup(struct mbuf *n, int len) return (m); bad: m_freem(n); - mbstat.m_mpfail++; /* XXX: No consistency. */ return (NULL); } @@ -1068,8 +844,6 @@ bad: * the amount of empty space before the data in the new mbuf to be specified * (in the event that the caller expects to prepend later). */ -int MSFail; - struct mbuf * m_copyup(struct mbuf *n, int len, int dstoff) { @@ -1078,12 +852,11 @@ m_copyup(struct mbuf *n, int len, int dstoff) if (len > (MHLEN - dstoff)) goto bad; - MGET(m, M_DONTWAIT, n->m_type); + m = m_get(M_NOWAIT, n->m_type); if (m == NULL) goto bad; - m->m_len = 0; if (n->m_flags & M_PKTHDR) - M_MOVE_PKTHDR(m, n); + m_move_pkthdr(m, n); m->m_data += dstoff; space = &m->m_dat[MLEN] - (m->m_data + m->m_len); do { @@ -1107,7 +880,6 @@ m_copyup(struct mbuf *n, int len, int dstoff) return (m); bad: m_freem(n); - MSFail++; return (NULL); } @@ -1133,8 +905,18 @@ m_split(struct mbuf *m0, int len0, int wait) if (m == NULL) return (NULL); remain = m->m_len - len; - if (m0->m_flags & M_PKTHDR) { - MGETHDR(n, wait, m0->m_type); + if (m0->m_flags & M_PKTHDR && remain == 0) { + n = m_gethdr(wait, m0->m_type); + if (n == NULL) + return (NULL); + n->m_next = m->m_next; + m->m_next = NULL; + n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; + n->m_pkthdr.len = m0->m_pkthdr.len - len0; + m0->m_pkthdr.len = len0; + return (n); + } else if (m0->m_flags & M_PKTHDR) { + n = m_gethdr(wait, m0->m_type); if (n == NULL) return (NULL); n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; @@ -1144,7 +926,7 @@ m_split(struct mbuf *m0, int len0, int wait) goto extpacket; if (remain > MHLEN) { /* m can't be the lead packet */ - MH_ALIGN(n, 0); + M_ALIGN(n, 0); n->m_next = m_split(m, len, wait); if (n->m_next == NULL) { (void) m_free(n); @@ -1154,13 +936,13 @@ m_split(struct mbuf *m0, int len0, int wait) return (n); } } else - MH_ALIGN(n, remain); + M_ALIGN(n, remain); } else if (remain == 0) { n = m->m_next; m->m_next = NULL; return (n); } else { - MGET(n, wait, m->m_type); + n = m_get(wait, m->m_type); if (n == NULL) return (NULL); M_ALIGN(n, remain); @@ -1197,10 +979,10 @@ m_devget(char *buf, int totlen, int off, struct ifnet *ifp, while (totlen > 0) { if (top == NULL) { /* First one, must be PKTHDR */ if (totlen + off >= MINCLSIZE) { - m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR); + m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); len = MCLBYTES; } else { - m = m_gethdr(M_DONTWAIT, MT_DATA); + m = m_gethdr(M_NOWAIT, MT_DATA); len = MHLEN; /* Place initial small packet/header at end of mbuf */ @@ -1215,10 +997,10 @@ m_devget(char *buf, int totlen, int off, struct ifnet *ifp, m->m_pkthdr.len = totlen; } else { if (totlen + off >= MINCLSIZE) { - m = m_getcl(M_DONTWAIT, MT_DATA, 0); + m = m_getcl(M_NOWAIT, MT_DATA, 0); len = MCLBYTES; } else { - m = m_get(M_DONTWAIT, MT_DATA); + m = m_get(M_NOWAIT, MT_DATA); len = MLEN; } if (m == NULL) { @@ -1262,7 +1044,7 @@ m_copyback(struct mbuf *m0, int off, int len, c_caddr_t cp) off -= mlen; totlen += mlen; if (m->m_next == NULL) { - n = m_get(M_DONTWAIT, m->m_type); + n = m_get(M_NOWAIT, m->m_type); if (n == NULL) goto out; bzero(mtod(n, caddr_t), MLEN); @@ -1286,7 +1068,7 @@ m_copyback(struct mbuf *m0, int off, int len, c_caddr_t cp) if (len == 0) break; if (m->m_next == NULL) { - n = m_get(M_DONTWAIT, m->m_type); + n = m_get(M_NOWAIT, m->m_type); if (n == NULL) break; n->m_len = min(MLEN, len); @@ -1330,7 +1112,7 @@ m_append(struct mbuf *m0, int len, c_caddr_t cp) * Allocate a new mbuf; could check space * and allocate a cluster instead. */ - n = m_get(M_DONTWAIT, m->m_type); + n = m_get(M_NOWAIT, m->m_type); if (n == NULL) break; n->m_len = min(MLEN, remainder); @@ -1498,7 +1280,7 @@ m_defrag(struct mbuf *m0, int how) goto nospace; } #endif - + if (m0->m_pkthdr.len > MHLEN) m_final = m_getcl(how, MT_DATA, M_PKTHDR); else @@ -1664,7 +1446,7 @@ m_fragment(struct mbuf *m0, int how, int length) if (!(m0->m_flags & M_PKTHDR)) return (m0); - + if ((length == 0) || (length < -2)) return (m0); @@ -1805,25 +1587,6 @@ m_mbuftouio(struct uio *uio, struct mbuf *m, int len) return (0); } -/* - * Set the m_data pointer of a newly-allocated mbuf - * to place an object of the specified size at the - * end of the mbuf, longword aligned. - */ -void -m_align(struct mbuf *m, int len) -{ - int adjust; - - if (m->m_flags & M_EXT) - adjust = m->m_ext.ext_size - len; - else if (m->m_flags & M_PKTHDR) - adjust = MHLEN - len; - else - adjust = MLEN - len; - m->m_data += adjust &~ (sizeof(long)-1); -} - /* * Create a writable copy of the mbuf chain. While doing this * we compact the chain with a goal of producing a chain with @@ -1859,7 +1622,7 @@ m_unshare(struct mbuf *m0, int how) m->m_len <= M_TRAILINGSPACE(mprev)) { /* XXX: this ignores mbuf types */ memcpy(mtod(mprev, caddr_t) + mprev->m_len, - mtod(m, caddr_t), m->m_len); + mtod(m, caddr_t), m->m_len); mprev->m_len += m->m_len; mprev->m_next = m->m_next; /* unlink from chain */ m_free(m); /* reclaim mbuf */ @@ -1891,7 +1654,7 @@ m_unshare(struct mbuf *m0, int how) m->m_len <= M_TRAILINGSPACE(mprev)) { /* XXX: this ignores mbuf types */ memcpy(mtod(mprev, caddr_t) + mprev->m_len, - mtod(m, caddr_t), m->m_len); + mtod(m, caddr_t), m->m_len); mprev->m_len += m->m_len; mprev->m_next = m->m_next; /* unlink from chain */ m_free(m); /* reclaim mbuf */ @@ -1902,43 +1665,23 @@ m_unshare(struct mbuf *m0, int how) } /* - * Allocate new space to hold the copy... - */ - /* XXX why can M_PKTHDR be set past the first mbuf? */ - if (mprev == NULL && (m->m_flags & M_PKTHDR)) { - /* - * NB: if a packet header is present we must - * allocate the mbuf separately from any cluster - * because M_MOVE_PKTHDR will smash the data - * pointer and drop the M_EXT marker. - */ - MGETHDR(n, how, m->m_type); - if (n == NULL) { - m_freem(m0); - return (NULL); - } - M_MOVE_PKTHDR(n, m); - MCLGET(n, how); - if ((n->m_flags & M_EXT) == 0) { - m_free(n); - m_freem(m0); - return (NULL); - } - } else { - n = m_getcl(how, m->m_type, m->m_flags); - if (n == NULL) { - m_freem(m0); - return (NULL); - } - } - /* - * ... and copy the data. We deal with jumbo mbufs - * (i.e. m_len > MCLBYTES) by splitting them into - * clusters. We could just malloc a buffer and make - * it external but too many device drivers don't know - * how to break up the non-contiguous memory when + * Allocate new space to hold the copy and copy the data. + * We deal with jumbo mbufs (i.e. m_len > MCLBYTES) by + * splitting them into clusters. We could just malloc a + * buffer and make it external but too many device drivers + * don't know how to break up the non-contiguous memory when * doing DMA. */ + n = m_getcl(how, m->m_type, m->m_flags & M_COPYFLAGS); + if (n == NULL) { + m_freem(m0); + return (NULL); + } + if (m->m_flags & M_PKTHDR) { + KASSERT(mprev == NULL, ("%s: m0 %p, m %p has M_PKTHDR", + __func__, m0, m)); + m_move_pkthdr(n, m); + } len = m->m_len; off = 0; mfirst = n; @@ -1949,7 +1692,7 @@ m_unshare(struct mbuf *m0, int how) n->m_len = cc; if (mlast != NULL) mlast->m_next = n; - mlast = n; + mlast = n; #if 0 newipsecstat.ips_clcopied++; #endif @@ -1959,14 +1702,14 @@ m_unshare(struct mbuf *m0, int how) break; off += cc; - n = m_getcl(how, m->m_type, m->m_flags); + n = m_getcl(how, m->m_type, m->m_flags & M_COPYFLAGS); if (n == NULL) { m_freem(mfirst); m_freem(m0); return (NULL); } } - n->m_next = m->m_next; + n->m_next = m->m_next; if (mprev == NULL) m0 = mfirst; /* new head of chain */ else @@ -2002,7 +1745,7 @@ m_profile(struct mbuf *m) int segments = 0; int used = 0; int wasted = 0; - + while (m) { segments++; used += m->m_len; @@ -2037,11 +1780,10 @@ mbprof_textify(void) int offset; char *c; uint64_t *p; - p = &mbprof.wasted[0]; c = mbprofbuf; - offset = snprintf(c, MP_MAXLINE + 10, + offset = snprintf(c, MP_MAXLINE + 10, "wasted:\n" "%ju %ju %ju %ju %ju %ju %ju %ju " "%ju %ju %ju %ju %ju %ju %ju %ju\n", @@ -2050,7 +1792,7 @@ mbprof_textify(void) #ifdef BIG_ARRAY p = &mbprof.wasted[16]; c += offset; - offset = snprintf(c, MP_MAXLINE, + offset = snprintf(c, MP_MAXLINE, "%ju %ju %ju %ju %ju %ju %ju %ju " "%ju %ju %ju %ju %ju %ju %ju %ju\n", p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], @@ -2058,7 +1800,7 @@ mbprof_textify(void) #endif p = &mbprof.used[0]; c += offset; - offset = snprintf(c, MP_MAXLINE + 10, + offset = snprintf(c, MP_MAXLINE + 10, "used:\n" "%ju %ju %ju %ju %ju %ju %ju %ju " "%ju %ju %ju %ju %ju %ju %ju %ju\n", @@ -2067,7 +1809,7 @@ mbprof_textify(void) #ifdef BIG_ARRAY p = &mbprof.used[16]; c += offset; - offset = snprintf(c, MP_MAXLINE, + offset = snprintf(c, MP_MAXLINE, "%ju %ju %ju %ju %ju %ju %ju %ju " "%ju %ju %ju %ju %ju %ju %ju %ju\n", p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], @@ -2075,7 +1817,7 @@ mbprof_textify(void) #endif p = &mbprof.segments[0]; c += offset; - offset = snprintf(c, MP_MAXLINE + 10, + offset = snprintf(c, MP_MAXLINE + 10, "segments:\n" "%ju %ju %ju %ju %ju %ju %ju %ju " "%ju %ju %ju %ju %ju %ju %ju %ju\n", @@ -2084,7 +1826,7 @@ mbprof_textify(void) #ifdef BIG_ARRAY p = &mbprof.segments[16]; c += offset; - offset = snprintf(c, MP_MAXLINE, + offset = snprintf(c, MP_MAXLINE, "%ju %ju %ju %ju %ju %ju %ju %ju " "%ju %ju %ju %ju %ju %ju %ju %jju", p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], @@ -2106,16 +1848,16 @@ static int mbprof_clr_handler(SYSCTL_HANDLER_ARGS) { int clear, error; - + clear = 0; error = sysctl_handle_int(oidp, &clear, 0, req); if (error || !req->newptr) return (error); - + if (clear) { bzero(&mbprof, sizeof(mbprof)); } - + return (error); } -- cgit v1.2.3