From c40e45b75eb76d79a05c7fa85c1fa9b5c728a12f Mon Sep 17 00:00:00 2001 From: Sebastian Huber Date: Fri, 7 Oct 2016 15:10:20 +0200 Subject: Update to FreeBSD head 2016-08-23 Git mirror commit 9fe7c416e6abb28b1398fd3e5687099846800cfd. --- freebsd/sys/kern/kern_mbuf.c | 609 +++++++++++++++++++++++++++++-------------- 1 file changed, 407 insertions(+), 202 deletions(-) (limited to 'freebsd/sys/kern/kern_mbuf.c') diff --git a/freebsd/sys/kern/kern_mbuf.c b/freebsd/sys/kern/kern_mbuf.c index 74e7aa10..6229b836 100644 --- a/freebsd/sys/kern/kern_mbuf.c +++ b/freebsd/sys/kern/kern_mbuf.c @@ -34,24 +34,24 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include #include #include #include +#include +#include #include #include #include -#include - #include #include #include #include #include #include -#include #include #ifdef __rtems__ #include @@ -97,7 +97,7 @@ __FBSDID("$FreeBSD$"); * * Whenever an object is allocated from the underlying global * memory pool it gets pre-initialized with the _zinit_ functions. - * When the Keg's are overfull objects get decomissioned with + * When the Keg's are overfull objects get decommissioned with * _zfini_ functions and free'd back to the global memory pool. * */ @@ -107,12 +107,11 @@ int nmbclusters; /* limits number of mbuf clusters */ int nmbjumbop; /* limits number of page size jumbo clusters */ int nmbjumbo9; /* limits number of 9k jumbo clusters */ int nmbjumbo16; /* limits number of 16k jumbo clusters */ -struct mbstat mbstat; static quad_t maxmbufmem; /* overall real memory limit for all mbufs */ -SYSCTL_QUAD(_kern_ipc, OID_AUTO, maxmbufmem, CTLFLAG_RDTUN, &maxmbufmem, 0, - "Maximum real memory allocateable to various mbuf types"); +SYSCTL_QUAD(_kern_ipc, OID_AUTO, maxmbufmem, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &maxmbufmem, 0, + "Maximum real memory allocatable to various mbuf types"); /* * tunable_mbinit() has to be run before any mbuf allocations are done. @@ -128,8 +127,7 @@ tunable_mbinit(void *dummy) * available kernel memory (physical or kmem). * At most it can be 3/4 of available kernel memory. */ - realmem = qmin((quad_t)physmem * PAGE_SIZE, - vm_map_max(kmem_map) - vm_map_min(kmem_map)); + realmem = qmin((quad_t)physmem * PAGE_SIZE, vm_kmem_size); maxmbufmem = realmem / 2; TUNABLE_QUAD_FETCH("kern.ipc.maxmbufmem", &maxmbufmem); if (maxmbufmem > realmem / 4 * 3) @@ -274,9 +272,6 @@ SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbufs, CTLTYPE_INT|CTLFLAG_RW, &nmbufs, 0, sysctl_nmbufs, "IU", "Maximum number of mbufs allowed"); -SYSCTL_STRUCT(_kern_ipc, OID_AUTO, mbstat, CTLFLAG_RD, &mbstat, mbstat, - "Mbuf general information and statistics"); - /* * Zones from which we allocate. */ @@ -286,7 +281,6 @@ uma_zone_t zone_pack; uma_zone_t zone_jumbop; uma_zone_t zone_jumbo9; uma_zone_t zone_jumbo16; -uma_zone_t zone_ext_refcnt; /* * Local prototypes. @@ -295,15 +289,13 @@ static int mb_ctor_mbuf(void *, int, void *, int); static int mb_ctor_clust(void *, int, void *, int); static int mb_ctor_pack(void *, int, void *, int); static void mb_dtor_mbuf(void *, int, void *); -static void mb_dtor_clust(void *, int, void *); static void mb_dtor_pack(void *, int, void *); static int mb_zinit_pack(void *, int, int); static void mb_zfini_pack(void *, int); +static void mb_reclaim(uma_zone_t, int); +static void *mbuf_jumbo_alloc(uma_zone_t, vm_size_t, uint8_t *, int); -static void mb_reclaim(void *); -static void *mbuf_jumbo_alloc(uma_zone_t, int, uint8_t *, int); - -/* Ensure that MSIZE doesn't break dtom() - it must be a power of 2 */ +/* Ensure that MSIZE is a power of 2. */ CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE); /* @@ -326,63 +318,66 @@ mbuf_init(void *dummy) MSIZE - 1, UMA_ZONE_MAXBUCKET); if (nmbufs > 0) nmbufs = uma_zone_set_max(zone_mbuf, nmbufs); + uma_zone_set_warning(zone_mbuf, "kern.ipc.nmbufs limit reached"); + uma_zone_set_maxaction(zone_mbuf, mb_reclaim); zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES, - mb_ctor_clust, mb_dtor_clust, + mb_ctor_clust, #ifdef INVARIANTS - trash_init, trash_fini, + trash_dtor, trash_init, trash_fini, #else - NULL, NULL, + NULL, NULL, NULL, #endif - UMA_ALIGN_PTR, UMA_ZONE_REFCNT); + UMA_ALIGN_PTR, 0); if (nmbclusters > 0) nmbclusters = uma_zone_set_max(zone_clust, nmbclusters); + uma_zone_set_warning(zone_clust, "kern.ipc.nmbclusters limit reached"); + uma_zone_set_maxaction(zone_clust, mb_reclaim); zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack, mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf); /* Make jumbo frame zone too. Page size, 9k and 16k. */ zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE, - mb_ctor_clust, mb_dtor_clust, + mb_ctor_clust, #ifdef INVARIANTS - trash_init, trash_fini, + trash_dtor, trash_init, trash_fini, #else - NULL, NULL, + NULL, NULL, NULL, #endif - UMA_ALIGN_PTR, UMA_ZONE_REFCNT); + UMA_ALIGN_PTR, 0); if (nmbjumbop > 0) nmbjumbop = uma_zone_set_max(zone_jumbop, nmbjumbop); + uma_zone_set_warning(zone_jumbop, "kern.ipc.nmbjumbop limit reached"); + uma_zone_set_maxaction(zone_jumbop, mb_reclaim); zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES, - mb_ctor_clust, mb_dtor_clust, + mb_ctor_clust, #ifdef INVARIANTS - trash_init, trash_fini, + trash_dtor, trash_init, trash_fini, #else - NULL, NULL, + NULL, NULL, NULL, #endif - UMA_ALIGN_PTR, UMA_ZONE_REFCNT); + UMA_ALIGN_PTR, 0); uma_zone_set_allocf(zone_jumbo9, mbuf_jumbo_alloc); if (nmbjumbo9 > 0) nmbjumbo9 = uma_zone_set_max(zone_jumbo9, nmbjumbo9); + uma_zone_set_warning(zone_jumbo9, "kern.ipc.nmbjumbo9 limit reached"); + uma_zone_set_maxaction(zone_jumbo9, mb_reclaim); zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES, - mb_ctor_clust, mb_dtor_clust, + mb_ctor_clust, #ifdef INVARIANTS - trash_init, trash_fini, + trash_dtor, trash_init, trash_fini, #else - NULL, NULL, + NULL, NULL, NULL, #endif - UMA_ALIGN_PTR, UMA_ZONE_REFCNT); + UMA_ALIGN_PTR, 0); uma_zone_set_allocf(zone_jumbo16, mbuf_jumbo_alloc); if (nmbjumbo16 > 0) nmbjumbo16 = uma_zone_set_max(zone_jumbo16, nmbjumbo16); - - zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int), - NULL, NULL, - NULL, NULL, - UMA_ALIGN_PTR, UMA_ZONE_ZINIT); - - /* uma_prealloc() goes here... */ + uma_zone_set_warning(zone_jumbo16, "kern.ipc.nmbjumbo16 limit reached"); + uma_zone_set_maxaction(zone_jumbo16, mb_reclaim); /* * Hook event handler for low-memory situation, used to @@ -393,25 +388,6 @@ mbuf_init(void *dummy) EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL, EVENTHANDLER_PRI_FIRST); #endif /* __rtems__ */ - - /* - * [Re]set counters and local statistics knobs. - * XXX Some of these should go and be replaced, but UMA stat - * gathering needs to be revised. - */ - mbstat.m_mbufs = 0; - mbstat.m_mclusts = 0; - mbstat.m_drain = 0; - mbstat.m_msize = MSIZE; - mbstat.m_mclbytes = MCLBYTES; - mbstat.m_minclsize = MINCLSIZE; - mbstat.m_mlen = MLEN; - mbstat.m_mhlen = MHLEN; - mbstat.m_numtypes = MT_NTYPES; - - mbstat.m_mcfail = mbstat.m_mpfail = 0; - mbstat.sf_iocnt = 0; - mbstat.sf_allocwait = mbstat.sf_allocfail = 0; } SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL); @@ -422,13 +398,13 @@ SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL); * pages. */ static void * -mbuf_jumbo_alloc(uma_zone_t zone, int bytes, uint8_t *flags, int wait) +mbuf_jumbo_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *flags, int wait) { /* Inform UMA that this allocator uses kernel_map/object. */ *flags = UMA_SLAB_KERNEL; #ifndef __rtems__ - return ((void *)kmem_alloc_contig(kernel_map, bytes, wait, + return ((void *)kmem_alloc_contig(kernel_arena, bytes, wait, (vm_paddr_t)0, ~(vm_paddr_t)0, 1, 0, VM_MEMATTR_DEFAULT)); #else /* __rtems__ */ return ((void *)malloc(bytes, M_TEMP, wait)); @@ -447,18 +423,14 @@ mb_ctor_mbuf(void *mem, int size, void *arg, int how) { struct mbuf *m; struct mb_args *args; -#ifdef MAC int error; -#endif int flags; short type; #ifdef INVARIANTS trash_ctor(mem, size, arg, how); #endif - m = (struct mbuf *)mem; args = (struct mb_args *)arg; - flags = args->flags; type = args->type; /* @@ -468,31 +440,13 @@ mb_ctor_mbuf(void *mem, int size, void *arg, int how) if (type == MT_NOINIT) return (0); - m->m_next = NULL; - m->m_nextpkt = NULL; - m->m_len = 0; - m->m_flags = flags; - m->m_type = type; - if (flags & M_PKTHDR) { - m->m_data = m->m_pktdat; - m->m_pkthdr.rcvif = NULL; - m->m_pkthdr.header = NULL; - m->m_pkthdr.len = 0; - m->m_pkthdr.csum_flags = 0; - m->m_pkthdr.csum_data = 0; - m->m_pkthdr.tso_segsz = 0; - m->m_pkthdr.ether_vtag = 0; - m->m_pkthdr.flowid = 0; - SLIST_INIT(&m->m_pkthdr.tags); -#ifdef MAC - /* If the label init fails, fail the alloc */ - error = mac_mbuf_init(m, how); - if (error) - return (error); -#endif - } else - m->m_data = m->m_dat; - return (0); + m = (struct mbuf *)mem; + flags = args->flags; + MPASS((flags & M_NOFREE) == 0); + + error = m_init(m, how, type, flags); + + return (error); } /* @@ -507,10 +461,9 @@ mb_dtor_mbuf(void *mem, int size, void *arg) m = (struct mbuf *)mem; flags = (unsigned long)arg; - if ((flags & MB_NOTAGS) == 0 && (m->m_flags & M_PKTHDR) != 0) - m_tag_delete_chain(m, NULL); - KASSERT((m->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__)); KASSERT((m->m_flags & M_NOFREE) == 0, ("%s: M_NOFREE set", __func__)); + if (!(flags & MB_DTOR_SKIP) && (m->m_flags & M_PKTHDR) && !SLIST_EMPTY(&m->m_pkthdr.tags)) + m_tag_delete_chain(m, NULL); #ifdef INVARIANTS trash_dtor(mem, size, arg); #endif @@ -536,7 +489,6 @@ mb_dtor_pack(void *mem, int size, void *arg) KASSERT(m->m_ext.ext_arg2 == NULL, ("%s: ext_arg2 != NULL", __func__)); KASSERT(m->m_ext.ext_size == MCLBYTES, ("%s: ext_size != MCLBYTES", __func__)); KASSERT(m->m_ext.ext_type == EXT_PACKET, ("%s: ext_type != EXT_PACKET", __func__)); - KASSERT(*m->m_ext.ref_cnt == 1, ("%s: ref_cnt != 1", __func__)); #ifdef INVARIANTS trash_dtor(m->m_ext.ext_buf, MCLBYTES, arg); #endif @@ -564,40 +516,11 @@ static int mb_ctor_clust(void *mem, int size, void *arg, int how) { struct mbuf *m; - u_int *refcnt; - int type; - uma_zone_t zone; #ifdef INVARIANTS trash_ctor(mem, size, arg, how); #endif - switch (size) { - case MCLBYTES: - type = EXT_CLUSTER; - zone = zone_clust; - break; -#if MJUMPAGESIZE != MCLBYTES - case MJUMPAGESIZE: - type = EXT_JUMBOP; - zone = zone_jumbop; - break; -#endif - case MJUM9BYTES: - type = EXT_JUMBO9; - zone = zone_jumbo9; - break; - case MJUM16BYTES: - type = EXT_JUMBO16; - zone = zone_jumbo16; - break; - default: - panic("unknown cluster size"); - break; - } - m = (struct mbuf *)arg; - refcnt = uma_find_refcnt(zone, mem); - *refcnt = 1; if (m != NULL) { m->m_ext.ext_buf = (caddr_t)mem; m->m_data = m->m_ext.ext_buf; @@ -606,31 +529,14 @@ mb_ctor_clust(void *mem, int size, void *arg, int how) m->m_ext.ext_arg1 = NULL; m->m_ext.ext_arg2 = NULL; m->m_ext.ext_size = size; - m->m_ext.ext_type = type; - m->m_ext.ref_cnt = refcnt; + m->m_ext.ext_type = m_gettype(size); + m->m_ext.ext_flags = EXT_FLAG_EMBREF; + m->m_ext.ext_count = 1; } return (0); } -/* - * The Mbuf Cluster zone destructor. - */ -static void -mb_dtor_clust(void *mem, int size, void *arg) -{ -#ifdef INVARIANTS - uma_zone_t zone; - - zone = m_getzone(size); - KASSERT(*(uma_find_refcnt(zone, mem)) <= 1, - ("%s: refcnt incorrect %u", __func__, - *(uma_find_refcnt(zone, mem))) ); - - trash_dtor(mem, size, arg); -#endif -} - /* * The Packet secondary zone's init routine, executed on the * object's transition from mbuf keg slab to zone cache. @@ -678,93 +584,392 @@ mb_ctor_pack(void *mem, int size, void *arg, int how) { struct mbuf *m; struct mb_args *args; -#ifdef MAC - int error; -#endif - int flags; + int error, flags; short type; m = (struct mbuf *)mem; args = (struct mb_args *)arg; flags = args->flags; type = args->type; + MPASS((flags & M_NOFREE) == 0); #ifdef INVARIANTS trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how); #endif - m->m_next = NULL; - m->m_nextpkt = NULL; - m->m_data = m->m_ext.ext_buf; - m->m_len = 0; - m->m_flags = (flags | M_EXT); - m->m_type = type; - - if (flags & M_PKTHDR) { - m->m_pkthdr.rcvif = NULL; - m->m_pkthdr.len = 0; - m->m_pkthdr.header = NULL; - m->m_pkthdr.csum_flags = 0; - m->m_pkthdr.csum_data = 0; - m->m_pkthdr.tso_segsz = 0; - m->m_pkthdr.ether_vtag = 0; - m->m_pkthdr.flowid = 0; - SLIST_INIT(&m->m_pkthdr.tags); -#ifdef MAC - /* If the label init fails, fail the alloc */ - error = mac_mbuf_init(m, how); - if (error) - return (error); -#endif - } - /* m_ext is already initialized. */ - return (0); -} + error = m_init(m, how, type, flags); -int -m_pkthdr_init(struct mbuf *m, int how) -{ -#ifdef MAC - int error; -#endif - m->m_data = m->m_pktdat; - SLIST_INIT(&m->m_pkthdr.tags); - m->m_pkthdr.rcvif = NULL; - m->m_pkthdr.header = NULL; - m->m_pkthdr.len = 0; - m->m_pkthdr.flowid = 0; - m->m_pkthdr.csum_flags = 0; - m->m_pkthdr.csum_data = 0; - m->m_pkthdr.tso_segsz = 0; - m->m_pkthdr.ether_vtag = 0; -#ifdef MAC - /* If the label init fails, fail the alloc */ - error = mac_mbuf_init(m, how); - if (error) - return (error); -#endif + /* m_ext is already initialized. */ + m->m_data = m->m_ext.ext_buf; + m->m_flags = (flags | M_EXT); - return (0); + return (error); } /* - * This is the protocol drain routine. + * This is the protocol drain routine. Called by UMA whenever any of the + * mbuf zones is closed to its limit. * * No locks should be held when this is called. The drain routines have to * presently acquire some locks which raises the possibility of lock order * reversal. */ static void -mb_reclaim(void *junk) +mb_reclaim(uma_zone_t zone __unused, int pending __unused) { struct domain *dp; struct protosw *pr; - WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL, - "mb_reclaim()"); + WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL, __func__); for (dp = domains; dp != NULL; dp = dp->dom_next) for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) if (pr->pr_drain != NULL) (*pr->pr_drain)(); } + +/* + * Clean up after mbufs with M_EXT storage attached to them if the + * reference count hits 1. + */ +void +mb_free_ext(struct mbuf *m) +{ + volatile u_int *refcnt; + struct mbuf *mref; + int freembuf; + + KASSERT(m->m_flags & M_EXT, ("%s: M_EXT not set on %p", __func__, m)); + + /* See if this is the mbuf that holds the embedded refcount. */ + if (m->m_ext.ext_flags & EXT_FLAG_EMBREF) { + refcnt = &m->m_ext.ext_count; + mref = m; + } else { + KASSERT(m->m_ext.ext_cnt != NULL, + ("%s: no refcounting pointer on %p", __func__, m)); + refcnt = m->m_ext.ext_cnt; + mref = __containerof(refcnt, struct mbuf, m_ext.ext_count); + } + + /* + * Check if the header is embedded in the cluster. It is + * important that we can't touch any of the mbuf fields + * after we have freed the external storage, since mbuf + * could have been embedded in it. For now, the mbufs + * embedded into the cluster are always of type EXT_EXTREF, + * and for this type we won't free the mref. + */ + if (m->m_flags & M_NOFREE) { + freembuf = 0; + KASSERT(m->m_ext.ext_type == EXT_EXTREF, + ("%s: no-free mbuf %p has wrong type", __func__, m)); + } else + freembuf = 1; + + /* Free attached storage if this mbuf is the only reference to it. */ + if (*refcnt == 1 || atomic_fetchadd_int(refcnt, -1) == 1) { + switch (m->m_ext.ext_type) { + case EXT_PACKET: + /* The packet zone is special. */ + if (*refcnt == 0) + *refcnt = 1; + uma_zfree(zone_pack, mref); + break; + case EXT_CLUSTER: + uma_zfree(zone_clust, m->m_ext.ext_buf); + uma_zfree(zone_mbuf, mref); + break; + case EXT_JUMBOP: + uma_zfree(zone_jumbop, m->m_ext.ext_buf); + uma_zfree(zone_mbuf, mref); + break; + case EXT_JUMBO9: + uma_zfree(zone_jumbo9, m->m_ext.ext_buf); + uma_zfree(zone_mbuf, mref); + break; + case EXT_JUMBO16: + uma_zfree(zone_jumbo16, m->m_ext.ext_buf); + uma_zfree(zone_mbuf, mref); + break; +#ifndef __rtems__ + case EXT_SFBUF: + sf_ext_free(m->m_ext.ext_arg1, m->m_ext.ext_arg2); + uma_zfree(zone_mbuf, mref); + break; + case EXT_SFBUF_NOCACHE: + sf_ext_free_nocache(m->m_ext.ext_arg1, + m->m_ext.ext_arg2); + uma_zfree(zone_mbuf, mref); + break; +#endif /* __rtems__ */ + case EXT_NET_DRV: + case EXT_MOD_TYPE: + case EXT_DISPOSABLE: + KASSERT(m->m_ext.ext_free != NULL, + ("%s: ext_free not set", __func__)); + (*(m->m_ext.ext_free))(m, m->m_ext.ext_arg1, + m->m_ext.ext_arg2); + uma_zfree(zone_mbuf, mref); + break; + case EXT_EXTREF: + KASSERT(m->m_ext.ext_free != NULL, + ("%s: ext_free not set", __func__)); + (*(m->m_ext.ext_free))(m, m->m_ext.ext_arg1, + m->m_ext.ext_arg2); + break; + default: + KASSERT(m->m_ext.ext_type == 0, + ("%s: unknown ext_type", __func__)); + } + } + + if (freembuf && m != mref) + uma_zfree(zone_mbuf, m); +} + +/* + * Official mbuf(9) allocation KPI for stack and drivers: + * + * m_get() - a single mbuf without any attachments, sys/mbuf.h. + * m_gethdr() - a single mbuf initialized as M_PKTHDR, sys/mbuf.h. + * m_getcl() - an mbuf + 2k cluster, sys/mbuf.h. + * m_clget() - attach cluster to already allocated mbuf. + * m_cljget() - attach jumbo cluster to already allocated mbuf. + * m_get2() - allocate minimum mbuf that would fit size argument. + * m_getm2() - allocate a chain of mbufs/clusters. + * m_extadd() - attach external cluster to mbuf. + * + * m_free() - free single mbuf with its tags and ext, sys/mbuf.h. + * m_freem() - free chain of mbufs. + */ + +int +m_clget(struct mbuf *m, int how) +{ + + KASSERT((m->m_flags & M_EXT) == 0, ("%s: mbuf %p has M_EXT", + __func__, m)); + m->m_ext.ext_buf = (char *)NULL; + uma_zalloc_arg(zone_clust, m, how); + /* + * On a cluster allocation failure, drain the packet zone and retry, + * we might be able to loosen a few clusters up on the drain. + */ + if ((how & M_NOWAIT) && (m->m_ext.ext_buf == NULL)) { + zone_drain(zone_pack); + uma_zalloc_arg(zone_clust, m, how); + } + MBUF_PROBE2(m__clget, m, how); + return (m->m_flags & M_EXT); +} + +/* + * m_cljget() is different from m_clget() as it can allocate clusters without + * attaching them to an mbuf. In that case the return value is the pointer + * to the cluster of the requested size. If an mbuf was specified, it gets + * the cluster attached to it and the return value can be safely ignored. + * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES. + */ +void * +m_cljget(struct mbuf *m, int how, int size) +{ + uma_zone_t zone; + void *retval; + + if (m != NULL) { + KASSERT((m->m_flags & M_EXT) == 0, ("%s: mbuf %p has M_EXT", + __func__, m)); + m->m_ext.ext_buf = NULL; + } + + zone = m_getzone(size); + retval = uma_zalloc_arg(zone, m, how); + + MBUF_PROBE4(m__cljget, m, how, size, retval); + + return (retval); +} + +/* + * m_get2() allocates minimum mbuf that would fit "size" argument. + */ +struct mbuf * +m_get2(int size, int how, short type, int flags) +{ + struct mb_args args; + struct mbuf *m, *n; + + args.flags = flags; + args.type = type; + + if (size <= MHLEN || (size <= MLEN && (flags & M_PKTHDR) == 0)) + return (uma_zalloc_arg(zone_mbuf, &args, how)); + if (size <= MCLBYTES) + return (uma_zalloc_arg(zone_pack, &args, how)); + + if (size > MJUMPAGESIZE) + return (NULL); + + m = uma_zalloc_arg(zone_mbuf, &args, how); + if (m == NULL) + return (NULL); + + n = uma_zalloc_arg(zone_jumbop, m, how); + if (n == NULL) { + uma_zfree(zone_mbuf, m); + return (NULL); + } + + return (m); +} + +/* + * m_getjcl() returns an mbuf with a cluster of the specified size attached. + * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES. + */ +struct mbuf * +m_getjcl(int how, short type, int flags, int size) +{ + struct mb_args args; + struct mbuf *m, *n; + uma_zone_t zone; + + if (size == MCLBYTES) + return m_getcl(how, type, flags); + + args.flags = flags; + args.type = type; + + m = uma_zalloc_arg(zone_mbuf, &args, how); + if (m == NULL) + return (NULL); + + zone = m_getzone(size); + n = uma_zalloc_arg(zone, m, how); + if (n == NULL) { + uma_zfree(zone_mbuf, m); + return (NULL); + } + return (m); +} + +/* + * Allocate a given length worth of mbufs and/or clusters (whatever fits + * best) and return a pointer to the top of the allocated chain. If an + * existing mbuf chain is provided, then we will append the new chain + * to the existing one but still return the top of the newly allocated + * chain. + */ +struct mbuf * +m_getm2(struct mbuf *m, int len, int how, short type, int flags) +{ + struct mbuf *mb, *nm = NULL, *mtail = NULL; + + KASSERT(len >= 0, ("%s: len is < 0", __func__)); + + /* Validate flags. */ + flags &= (M_PKTHDR | M_EOR); + + /* Packet header mbuf must be first in chain. */ + if ((flags & M_PKTHDR) && m != NULL) + flags &= ~M_PKTHDR; + + /* Loop and append maximum sized mbufs to the chain tail. */ + while (len > 0) { + if (len > MCLBYTES) + mb = m_getjcl(how, type, (flags & M_PKTHDR), + MJUMPAGESIZE); + else if (len >= MINCLSIZE) + mb = m_getcl(how, type, (flags & M_PKTHDR)); + else if (flags & M_PKTHDR) + mb = m_gethdr(how, type); + else + mb = m_get(how, type); + + /* Fail the whole operation if one mbuf can't be allocated. */ + if (mb == NULL) { + if (nm != NULL) + m_freem(nm); + return (NULL); + } + + /* Book keeping. */ + len -= M_SIZE(mb); + if (mtail != NULL) + mtail->m_next = mb; + else + nm = mb; + mtail = mb; + flags &= ~M_PKTHDR; /* Only valid on the first mbuf. */ + } + if (flags & M_EOR) + mtail->m_flags |= M_EOR; /* Only valid on the last mbuf. */ + + /* If mbuf was supplied, append new chain to the end of it. */ + if (m != NULL) { + for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next) + ; + mtail->m_next = nm; + mtail->m_flags &= ~M_EOR; + } else + m = nm; + + return (m); +} + +/*- + * Configure a provided mbuf to refer to the provided external storage + * buffer and setup a reference count for said buffer. + * + * Arguments: + * mb The existing mbuf to which to attach the provided buffer. + * buf The address of the provided external storage buffer. + * size The size of the provided buffer. + * freef A pointer to a routine that is responsible for freeing the + * provided external storage buffer. + * args A pointer to an argument structure (of any type) to be passed + * to the provided freef routine (may be NULL). + * flags Any other flags to be passed to the provided mbuf. + * type The type that the external storage buffer should be + * labeled with. + * + * Returns: + * Nothing. + */ +void +m_extadd(struct mbuf *mb, caddr_t buf, u_int size, + void (*freef)(struct mbuf *, void *, void *), void *arg1, void *arg2, + int flags, int type) +{ + + KASSERT(type != EXT_CLUSTER, ("%s: EXT_CLUSTER not allowed", __func__)); + + mb->m_flags |= (M_EXT | flags); + mb->m_ext.ext_buf = buf; + mb->m_data = mb->m_ext.ext_buf; + mb->m_ext.ext_size = size; + mb->m_ext.ext_free = freef; + mb->m_ext.ext_arg1 = arg1; + mb->m_ext.ext_arg2 = arg2; + mb->m_ext.ext_type = type; + + if (type != EXT_EXTREF) { + mb->m_ext.ext_count = 1; + mb->m_ext.ext_flags = EXT_FLAG_EMBREF; + } else + mb->m_ext.ext_flags = 0; +} + +/* + * Free an entire chain of mbufs and associated external buffers, if + * applicable. + */ +void +m_freem(struct mbuf *mb) +{ + + MBUF_PROBE1(m__freem, mb); + while (mb != NULL) + mb = m_free(mb); +} -- cgit v1.2.3