1 files changed, 231 insertions, 105 deletions

diff --git a/freebsd/sys/netinet6/in6_mcast.c b/freebsd/sys/netinet6/in6_mcast.c
index a634b18b..32660c89 100644
--- a/freebsd/sys/netinet6/in6_mcast.c
+++ b/freebsd/sys/netinet6/in6_mcast.c
@@ -43,13 +43,13 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
+#include <sys/gtaskqueue.h>
 #include <sys/kernel.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
 #include <sys/protosw.h>
 #include <sys/socket.h>
 #include <sys/socketvar.h>
-#include <sys/protosw.h>
 #include <sys/sysctl.h>
 #include <sys/priv.h>
 #include <sys/ktr.h>
@@ -61,8 +61,12 @@ __FBSDID("$FreeBSD$");
 #include <net/route.h>
 #include <net/vnet.h>
 
+
 #include <netinet/in.h>
+#include <netinet/udp.h>
 #include <netinet/in_var.h>
+#include <netinet/ip_var.h>
+#include <netinet/udp_var.h>
 #include <netinet6/in6_fib.h>
 #include <netinet6/in6_var.h>
 #include <netinet/ip6.h>
@@ -91,7 +95,7 @@ typedef union sockunion sockunion_t;
 
 static MALLOC_DEFINE(M_IN6MFILTER, "in6_mfilter",
     "IPv6 multicast PCB-layer source filter");
-static MALLOC_DEFINE(M_IP6MADDR, "in6_multi", "IPv6 multicast group");
+MALLOC_DEFINE(M_IP6MADDR, "in6_multi", "IPv6 multicast group");
 static MALLOC_DEFINE(M_IP6MOPTS, "ip6_moptions", "IPv6 multicast options");
 static MALLOC_DEFINE(M_IP6MSOURCE, "ip6_msource",
     "IPv6 multicast MLD-layer source filter");
@@ -109,8 +113,16 @@ RB_GENERATE(ip6_msource_tree, ip6_msource, im6s_link, ip6_msource_cmp);
  * any need for in6_multi itself to be virtualized -- it is bound to an ifp
  * anyway no matter what happens.
  */
-struct mtx in6_multi_mtx;
-MTX_SYSINIT(in6_multi_mtx, &in6_multi_mtx, "in6_multi_mtx", MTX_DEF);
+struct mtx in6_multi_list_mtx;
+MTX_SYSINIT(in6_multi_mtx, &in6_multi_list_mtx, "in6_multi_list_mtx", MTX_DEF);
+
+struct mtx in6_multi_free_mtx;
+MTX_SYSINIT(in6_multi_free_mtx, &in6_multi_free_mtx, "in6_multi_free_mtx", MTX_DEF);
+
+struct sx in6_multi_sx;
+SX_SYSINIT(in6_multi_sx, &in6_multi_sx, "in6_multi_sx");
+
+
 
 static void	im6f_commit(struct in6_mfilter *);
 static int	im6f_get_source(struct in6_mfilter *imf,
@@ -132,7 +144,7 @@ static struct in6_msource *
 		    const struct sockaddr *);
 static void	im6s_merge(struct ip6_msource *ims,
 		    const struct in6_msource *lims, const int rollback);
-static int	in6_mc_get(struct ifnet *, const struct in6_addr *,
+static int	in6_getmulti(struct ifnet *, const struct in6_addr *,
 		    struct in6_multi **);
 static int	in6m_get_source(struct in6_multi *inm,
 		    const struct in6_addr *addr, const int noalloc,
@@ -180,6 +192,7 @@ static SYSCTL_NODE(_net_inet6_ip6_mcast, OID_AUTO, filters,
     CTLFLAG_RD | CTLFLAG_MPSAFE, sysctl_ip6_mcast_filters,
     "Per-interface stack-wide source filters");
 
+int ifma6_restart = 0;
 #ifdef KTR
 /*
  * Inline function which wraps assertions for a valid ifp.
@@ -391,7 +404,7 @@ im6o_mc_filter(const struct ip6_moptions *imo, const struct ifnet *ifp,
  * Return 0 if successful, otherwise return an appropriate error code.
  */
 static int
-in6_mc_get(struct ifnet *ifp, const struct in6_addr *group,
+in6_getmulti(struct ifnet *ifp, const struct in6_addr *group,
     struct in6_multi **pinm)
 {
 	struct sockaddr_in6	 gsin6;
@@ -407,8 +420,8 @@ in6_mc_get(struct ifnet *ifp, const struct in6_addr *group,
 	 * re-acquire around the call.
 	 */
 	IN6_MULTI_LOCK_ASSERT();
+	IN6_MULTI_LIST_LOCK();
 	IF_ADDR_WLOCK(ifp);
-
 	inm = in6m_lookup_locked(ifp, group);
 	if (inm != NULL) {
 		/*
@@ -417,7 +430,7 @@ in6_mc_get(struct ifnet *ifp, const struct in6_addr *group,
 		 */
 		KASSERT(inm->in6m_refcount >= 1,
 		    ("%s: bad refcount %d", __func__, inm->in6m_refcount));
-		++inm->in6m_refcount;
+		in6m_acquire_locked(inm);
 		*pinm = inm;
 		goto out_locked;
 	}
@@ -431,10 +444,12 @@ in6_mc_get(struct ifnet *ifp, const struct in6_addr *group,
 	 * Check if a link-layer group is already associated
 	 * with this network-layer group on the given ifnet.
 	 */
+	IN6_MULTI_LIST_UNLOCK();
 	IF_ADDR_WUNLOCK(ifp);
 	error = if_addmulti(ifp, (struct sockaddr *)&gsin6, &ifma);
 	if (error != 0)
 		return (error);
+	IN6_MULTI_LIST_LOCK();
 	IF_ADDR_WLOCK(ifp);
 
 	/*
@@ -457,7 +472,7 @@ in6_mc_get(struct ifnet *ifp, const struct in6_addr *group,
 			panic("%s: ifma %p is inconsistent with %p (%p)",
 			    __func__, ifma, inm, group);
 #endif
-		++inm->in6m_refcount;
+		in6m_acquire_locked(inm);
 		*pinm = inm;
 		goto out_locked;
 	}
@@ -474,6 +489,7 @@ in6_mc_get(struct ifnet *ifp, const struct in6_addr *group,
 	 */
 	inm = malloc(sizeof(*inm), M_IP6MADDR, M_NOWAIT | M_ZERO);
 	if (inm == NULL) {
+		IN6_MULTI_LIST_UNLOCK();
 		IF_ADDR_WUNLOCK(ifp);
 		if_delmulti_ifma(ifma);
 		return (ENOMEM);
@@ -493,7 +509,8 @@ in6_mc_get(struct ifnet *ifp, const struct in6_addr *group,
 	ifma->ifma_protospec = inm;
 	*pinm = inm;
 
-out_locked:
+ out_locked:
+	IN6_MULTI_LIST_UNLOCK();
 	IF_ADDR_WUNLOCK(ifp);
 	return (error);
 }
@@ -504,36 +521,139 @@ out_locked:
  * If the refcount drops to 0, free the in6_multi record and
  * delete the underlying link-layer membership.
  */
-void
-in6m_release_locked(struct in6_multi *inm)
+static void
+in6m_release(struct in6_multi *inm)
 {
 	struct ifmultiaddr *ifma;
-
-	IN6_MULTI_LOCK_ASSERT();
+	struct ifnet *ifp;
 
 	CTR2(KTR_MLD, "%s: refcount is %d", __func__, inm->in6m_refcount);
 
-	if (--inm->in6m_refcount > 0) {
-		CTR2(KTR_MLD, "%s: refcount is now %d", __func__,
-		    inm->in6m_refcount);
-		return;
-	}
-
+	MPASS(inm->in6m_refcount == 0);
 	CTR2(KTR_MLD, "%s: freeing inm %p", __func__, inm);
 
 	ifma = inm->in6m_ifma;
+	ifp = inm->in6m_ifp;
+	MPASS(ifma->ifma_llifma == NULL);
 
 	/* XXX this access is not covered by IF_ADDR_LOCK */
 	CTR2(KTR_MLD, "%s: purging ifma %p", __func__, ifma);
-	KASSERT(ifma->ifma_protospec == inm,
-	    ("%s: ifma_protospec != inm", __func__));
-	ifma->ifma_protospec = NULL;
+	KASSERT(ifma->ifma_protospec == NULL,
+	    ("%s: ifma_protospec != NULL", __func__));
 
-	in6m_purge(inm);
+	if (ifp != NULL) {
+		CURVNET_SET(ifp->if_vnet);
+		in6m_purge(inm);
+		free(inm, M_IP6MADDR);
+		if_delmulti_ifma_flags(ifma, 1);
+		CURVNET_RESTORE();
+		if_rele(ifp);
+	} else {
+		in6m_purge(inm);
+		free(inm, M_IP6MADDR);
+		if_delmulti_ifma_flags(ifma, 1);
+	}
+}
+
+static struct grouptask free_gtask;
+static struct in6_multi_head in6m_free_list;
+static void in6m_release_task(void *arg __unused);
+static void in6m_init(void)
+{
+	SLIST_INIT(&in6m_free_list);
+	taskqgroup_config_gtask_init(NULL, &free_gtask, in6m_release_task, "in6m release task");
+}
 
-	free(inm, M_IP6MADDR);
+SYSINIT(in6m_init, SI_SUB_SMP + 1, SI_ORDER_FIRST,
+	in6m_init, NULL);
 
-	if_delmulti_ifma(ifma);
+
+void
+in6m_release_list_deferred(struct in6_multi_head *inmh)
+{
+	if (SLIST_EMPTY(inmh))
+		return;
+	mtx_lock(&in6_multi_free_mtx);
+	SLIST_CONCAT(&in6m_free_list, inmh, in6_multi, in6m_nrele);
+	mtx_unlock(&in6_multi_free_mtx);
+	GROUPTASK_ENQUEUE(&free_gtask);
+}
+
+void
+in6m_disconnect(struct in6_multi *inm)
+{
+	struct ifnet *ifp;
+	struct ifaddr *ifa;
+	struct in6_ifaddr *ifa6;
+	struct in6_multi_mship *imm, *imm_tmp;
+	struct ifmultiaddr *ifma, *ll_ifma;
+
+	ifp = inm->in6m_ifp;
+	IF_ADDR_WLOCK_ASSERT(ifp);
+	ifma = inm->in6m_ifma;
+
+	if_ref(ifp);
+	CK_STAILQ_REMOVE(&ifp->if_multiaddrs, ifma, ifmultiaddr, ifma_link);
+	MCDPRINTF("removed ifma: %p from %s\n", ifma, ifp->if_xname);
+	if ((ll_ifma = ifma->ifma_llifma) != NULL) {
+		MPASS(ifma != ll_ifma);
+		ifma->ifma_llifma = NULL;
+		MPASS(ll_ifma->ifma_llifma == NULL);
+		MPASS(ll_ifma->ifma_ifp == ifp);
+		if (--ll_ifma->ifma_refcount == 0) {
+			ifma6_restart = true;
+			CK_STAILQ_REMOVE(&ifp->if_multiaddrs, ll_ifma, ifmultiaddr, ifma_link);
+			MCDPRINTF("removed ll_ifma: %p from %s\n", ll_ifma, ifp->if_xname);
+			if_freemulti(ll_ifma);
+		}
+	}
+	CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
+		if (ifa->ifa_addr->sa_family != AF_INET6)
+			continue;
+		ifa6 = (void *)ifa;
+		LIST_FOREACH_SAFE(imm, &ifa6->ia6_memberships,
+		    i6mm_chain, imm_tmp) {
+			if (inm == imm->i6mm_maddr) {
+				LIST_REMOVE(imm, i6mm_chain);
+				free(imm, M_IP6MADDR);
+			}
+		}
+	}
+}
+
+void
+in6m_release_deferred(struct in6_multi *inm)
+{
+	struct in6_multi_head tmp;
+
+	IN6_MULTI_LIST_LOCK_ASSERT();
+	KASSERT(inm->in6m_refcount > 0, ("refcount == %d inm: %p", inm->in6m_refcount, inm));
+	if (--inm->in6m_refcount == 0) {
+		in6m_disconnect(inm);
+		SLIST_INIT(&tmp);
+		inm->in6m_ifma->ifma_protospec = NULL;
+		MPASS(inm->in6m_ifma->ifma_llifma == NULL);
+		SLIST_INSERT_HEAD(&tmp, inm, in6m_nrele);
+		in6m_release_list_deferred(&tmp);
+	}
+}
+
+static void
+in6m_release_task(void *arg __unused)
+{
+	struct in6_multi_head in6m_free_tmp;
+	struct in6_multi *inm, *tinm;
+
+	SLIST_INIT(&in6m_free_tmp);
+	mtx_lock(&in6_multi_free_mtx);
+	SLIST_CONCAT(&in6m_free_tmp, &in6m_free_list, in6_multi, in6m_nrele);
+	mtx_unlock(&in6_multi_free_mtx);
+	IN6_MULTI_LOCK();
+	SLIST_FOREACH_SAFE(inm, &in6m_free_tmp, in6m_nrele, tinm) {
+		SLIST_REMOVE_HEAD(&in6m_free_tmp, in6m_nrele);
+		in6m_release(inm);
+	}
+	IN6_MULTI_UNLOCK();
 }
 
 /*
@@ -546,7 +666,7 @@ in6m_clear_recorded(struct in6_multi *inm)
 {
 	struct ip6_msource	*ims;
 
-	IN6_MULTI_LOCK_ASSERT();
+	IN6_MULTI_LIST_LOCK_ASSERT();
 
 	RB_FOREACH(ims, ip6_msource_tree, &inm->in6m_srcs) {
 		if (ims->im6s_stp) {
@@ -586,7 +706,7 @@ in6m_record_source(struct in6_multi *inm, const struct in6_addr *addr)
 	struct ip6_msource	 find;
 	struct ip6_msource	*ims, *nims;
 
-	IN6_MULTI_LOCK_ASSERT();
+	IN6_MULTI_LIST_LOCK_ASSERT();
 
 	find.im6s_addr = *addr;
 	ims = RB_FIND(ip6_msource_tree, &inm->in6m_srcs, &find);
@@ -913,6 +1033,7 @@ in6m_merge(struct in6_multi *inm, /*const*/ struct in6_mfilter *imf)
 	schanged = 0;
 	error = 0;
 	nsrc1 = nsrc0 = 0;
+	IN6_MULTI_LIST_LOCK_ASSERT();
 
 	/*
 	 * Update the source filters first, as this may fail.
@@ -1089,65 +1210,16 @@ in6m_purge(struct in6_multi *inm)
  *
  * SMPng: Assume no mc locks held by caller.
  */
-struct in6_multi_mship *
-in6_joingroup(struct ifnet *ifp, struct in6_addr *mcaddr,
-    int *errorp, int delay)
-{
-	struct in6_multi_mship *imm;
-	int error;
-
-	imm = malloc(sizeof(*imm), M_IP6MADDR, M_NOWAIT);
-	if (imm == NULL) {
-		*errorp = ENOBUFS;
-		return (NULL);
-	}
-
-	delay = (delay * PR_FASTHZ) / hz;
-
-	error = in6_mc_join(ifp, mcaddr, NULL, &imm->i6mm_maddr, delay);
-	if (error) {
-		*errorp = error;
-		free(imm, M_IP6MADDR);
-		return (NULL);
-	}
-
-	return (imm);
-}
-
-/*
- * Leave a multicast address w/o sources.
- * KAME compatibility entry point.
- *
- * SMPng: Assume no mc locks held by caller.
- */
-int
-in6_leavegroup(struct in6_multi_mship *imm)
-{
-
-	if (imm->i6mm_maddr != NULL)
-		in6_mc_leave(imm->i6mm_maddr, NULL);
-	free(imm,  M_IP6MADDR);
-	return 0;
-}
-
-/*
- * Join a multicast group; unlocked entry point.
- *
- * SMPng: XXX: in6_mc_join() is called from in6_control() when upper
- * locks are not held. Fortunately, ifp is unlikely to have been detached
- * at this point, so we assume it's OK to recurse.
- */
 int
-in6_mc_join(struct ifnet *ifp, const struct in6_addr *mcaddr,
+in6_joingroup(struct ifnet *ifp, const struct in6_addr *mcaddr,
     /*const*/ struct in6_mfilter *imf, struct in6_multi **pinm,
     const int delay)
 {
 	int error;
 
 	IN6_MULTI_LOCK();
-	error = in6_mc_join_locked(ifp, mcaddr, imf, pinm, delay);
+	error = in6_joingroup_locked(ifp, mcaddr, NULL, pinm, delay);
 	IN6_MULTI_UNLOCK();
-
 	return (error);
 }
 
@@ -1161,12 +1233,13 @@ in6_mc_join(struct ifnet *ifp, const struct in6_addr *mcaddr,
  * code is returned.
  */
 int
-in6_mc_join_locked(struct ifnet *ifp, const struct in6_addr *mcaddr,
+in6_joingroup_locked(struct ifnet *ifp, const struct in6_addr *mcaddr,
     /*const*/ struct in6_mfilter *imf, struct in6_multi **pinm,
     const int delay)
 {
 	struct in6_mfilter	 timf;
 	struct in6_multi	*inm;
+	struct ifmultiaddr *ifma;
 	int			 error;
 #ifdef KTR
 	char			 ip6tbuf[INET6_ADDRSTRLEN];
@@ -1187,6 +1260,7 @@ in6_mc_join_locked(struct ifnet *ifp, const struct in6_addr *mcaddr,
 #endif
 
 	IN6_MULTI_LOCK_ASSERT();
+	IN6_MULTI_LIST_UNLOCK_ASSERT();
 
 	CTR4(KTR_MLD, "%s: join %s on %p(%s))", __func__,
 	    ip6_sprintf(ip6tbuf, mcaddr), ifp, if_name(ifp));
@@ -1202,13 +1276,13 @@ in6_mc_join_locked(struct ifnet *ifp, const struct in6_addr *mcaddr,
 		im6f_init(&timf, MCAST_UNDEFINED, MCAST_EXCLUDE);
 		imf = &timf;
 	}
-
-	error = in6_mc_get(ifp, mcaddr, &inm);
+	error = in6_getmulti(ifp, mcaddr, &inm);
 	if (error) {
-		CTR1(KTR_MLD, "%s: in6_mc_get() failure", __func__);
+		CTR1(KTR_MLD, "%s: in6_getmulti() failure", __func__);
 		return (error);
 	}
 
+	IN6_MULTI_LIST_LOCK();
 	CTR1(KTR_MLD, "%s: merge inm state", __func__);
 	error = in6m_merge(inm, imf);
 	if (error) {
@@ -1226,11 +1300,19 @@ in6_mc_join_locked(struct ifnet *ifp, const struct in6_addr *mcaddr,
 out_in6m_release:
 	if (error) {
 		CTR2(KTR_MLD, "%s: dropping ref on %p", __func__, inm);
-		in6m_release_locked(inm);
+		IF_ADDR_RLOCK(ifp);
+		CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
+			if (ifma->ifma_protospec == inm) {
+				ifma->ifma_protospec = NULL;
+				break;
+			}
+		}
+		in6m_release_deferred(inm);
+		IF_ADDR_RUNLOCK(ifp);
 	} else {
 		*pinm = inm;
 	}
-
+	IN6_MULTI_LIST_UNLOCK();
 	return (error);
 }
 
@@ -1238,14 +1320,13 @@ out_in6m_release:
  * Leave a multicast group; unlocked entry point.
  */
 int
-in6_mc_leave(struct in6_multi *inm, /*const*/ struct in6_mfilter *imf)
+in6_leavegroup(struct in6_multi *inm, /*const*/ struct in6_mfilter *imf)
 {
 	int error;
 
 	IN6_MULTI_LOCK();
-	error = in6_mc_leave_locked(inm, imf);
+	error = in6_leavegroup_locked(inm, imf);
 	IN6_MULTI_UNLOCK();
-
 	return (error);
 }
 
@@ -1263,9 +1344,10 @@ in6_mc_leave(struct in6_multi *inm, /*const*/ struct in6_mfilter *imf)
  * makes a state change downcall into MLD.
  */
 int
-in6_mc_leave_locked(struct in6_multi *inm, /*const*/ struct in6_mfilter *imf)
+in6_leavegroup_locked(struct in6_multi *inm, /*const*/ struct in6_mfilter *imf)
 {
 	struct in6_mfilter	 timf;
+	struct ifnet *ifp;
 	int			 error;
 #ifdef KTR
 	char			 ip6tbuf[INET6_ADDRSTRLEN];
@@ -1296,6 +1378,9 @@ in6_mc_leave_locked(struct in6_multi *inm, /*const*/ struct in6_mfilter *imf)
 	 * to be allocated, and there is no opportunity to roll back
 	 * the transaction, it MUST NOT fail.
 	 */
+
+	ifp = inm->in6m_ifp;
+	IN6_MULTI_LIST_LOCK();
 	CTR1(KTR_MLD, "%s: merge inm state", __func__);
 	error = in6m_merge(inm, imf);
 	KASSERT(error == 0, ("%s: failed to merge inm state", __func__));
@@ -1306,11 +1391,17 @@ in6_mc_leave_locked(struct in6_multi *inm, /*const*/ struct in6_mfilter *imf)
 		CTR1(KTR_MLD, "%s: failed mld downcall", __func__);
 
 	CTR2(KTR_MLD, "%s: dropping ref on %p", __func__, inm);
-	in6m_release_locked(inm);
+	if (ifp)
+		IF_ADDR_WLOCK(ifp);
+	in6m_release_deferred(inm);
+	if (ifp)
+		IF_ADDR_WUNLOCK(ifp);
+	IN6_MULTI_LIST_UNLOCK();
 
 	return (error);
 }
 
+
 /*
  * Block or unblock an ASM multicast source on an inpcb.
  * This implements the delta-based API described in RFC 3678.
@@ -1448,8 +1539,7 @@ in6p_block_unblock_source(struct inpcb *inp, struct sockopt *sopt)
 	/*
 	 * Begin state merge transaction at MLD layer.
 	 */
-	IN6_MULTI_LOCK();
-
+	IN6_MULTI_LIST_LOCK();
 	CTR1(KTR_MLD, "%s: merge inm state", __func__);
 	error = in6m_merge(inm, imf);
 	if (error)
@@ -1461,7 +1551,7 @@ in6p_block_unblock_source(struct inpcb *inp, struct sockopt *sopt)
 			CTR1(KTR_MLD, "%s: failed mld downcall", __func__);
 	}
 
-	IN6_MULTI_UNLOCK();
+	IN6_MULTI_LIST_UNLOCK();
 
 out_im6f_rollback:
 	if (error)
@@ -1530,22 +1620,36 @@ in6p_findmoptions(struct inpcb *inp)
  * Discard the IPv6 multicast options (and source filters).
  *
  * SMPng: NOTE: assumes INP write lock is held.
+ *
+ * XXX can all be safely deferred to epoch_call
+ *
  */
-void
-ip6_freemoptions(struct ip6_moptions *imo)
+
+static void
+inp_gcmoptions(epoch_context_t ctx)
 {
+	struct ip6_moptions *imo;
 	struct in6_mfilter	*imf;
+	struct in6_multi *inm;
+	struct ifnet *ifp;
 	size_t			 idx, nmships;
 
-	KASSERT(imo != NULL, ("%s: ip6_moptions is NULL", __func__));
+	imo =  __containerof(ctx, struct ip6_moptions, imo6_epoch_ctx);
 
 	nmships = imo->im6o_num_memberships;
 	for (idx = 0; idx < nmships; ++idx) {
 		imf = imo->im6o_mfilters ? &imo->im6o_mfilters[idx] : NULL;
 		if (imf)
 			im6f_leave(imf);
-		/* XXX this will thrash the lock(s) */
-		(void)in6_mc_leave(imo->im6o_membership[idx], imf);
+		inm = imo->im6o_membership[idx];
+		ifp = inm->in6m_ifp;
+		if (ifp != NULL) {
+			CURVNET_SET(ifp->if_vnet);
+			(void)in6_leavegroup(inm, imf);
+			CURVNET_RESTORE();
+		} else {
+			(void)in6_leavegroup(inm, imf);
+		}
 		if (imf)
 			im6f_purge(imf);
 	}
@@ -1556,6 +1660,14 @@ ip6_freemoptions(struct ip6_moptions *imo)
 	free(imo, M_IP6MOPTS);
 }
 
+void
+ip6_freemoptions(struct ip6_moptions *imo)
+{
+	if (imo == NULL)
+		return;
+	epoch_call(net_epoch_preempt, &imo->imo6_epoch_ctx, inp_gcmoptions);
+}
+
 /*
  * Atomically get source filters on a socket for an IPv6 multicast group.
  * Called with INP lock held; returns with lock released.
@@ -2036,10 +2148,12 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt)
 	/*
 	 * Begin state merge transaction at MLD layer.
 	 */
+	in_pcbref(inp);
+	INP_WUNLOCK(inp);
 	IN6_MULTI_LOCK();
 
 	if (is_new) {
-		error = in6_mc_join_locked(ifp, &gsa->sin6.sin6_addr, imf,
+		error = in6_joingroup_locked(ifp, &gsa->sin6.sin6_addr, imf,
 		    &inm, 0);
 		if (error) {
 			IN6_MULTI_UNLOCK();
@@ -2048,6 +2162,7 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt)
 		imo->im6o_membership[idx] = inm;
 	} else {
 		CTR1(KTR_MLD, "%s: merge inm state", __func__);
+		IN6_MULTI_LIST_LOCK();
 		error = in6m_merge(inm, imf);
 		if (error)
 			CTR1(KTR_MLD, "%s: failed to merge inm state",
@@ -2059,10 +2174,13 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt)
 				CTR1(KTR_MLD, "%s: failed mld downcall",
 				    __func__);
 		}
+		IN6_MULTI_LIST_UNLOCK();
 	}
 
 	IN6_MULTI_UNLOCK();
-	INP_WLOCK_ASSERT(inp);
+	INP_WLOCK(inp);
+	if (in_pcbrele_wlocked(inp))
+		return (ENXIO);
 	if (error) {
 		im6f_rollback(imf);
 		if (is_new)
@@ -2277,6 +2395,8 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt)
 	/*
 	 * Begin state merge transaction at MLD layer.
 	 */
+	in_pcbref(inp);
+	INP_WUNLOCK(inp);
 	IN6_MULTI_LOCK();
 
 	if (is_final) {
@@ -2284,9 +2404,10 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt)
 		 * Give up the multicast address record to which
 		 * the membership points.
 		 */
-		(void)in6_mc_leave_locked(inm, imf);
+		(void)in6_leavegroup_locked(inm, imf);
 	} else {
 		CTR1(KTR_MLD, "%s: merge inm state", __func__);
+		IN6_MULTI_LIST_LOCK();
 		error = in6m_merge(inm, imf);
 		if (error)
 			CTR1(KTR_MLD, "%s: failed to merge inm state",
@@ -2298,9 +2419,13 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt)
 				CTR1(KTR_MLD, "%s: failed mld downcall",
 				    __func__);
 		}
+		IN6_MULTI_LIST_UNLOCK();
 	}
 
 	IN6_MULTI_UNLOCK();
+	INP_WLOCK(inp);
+	if (in_pcbrele_wlocked(inp))
+		return (ENXIO);
 
 	if (error)
 		im6f_rollback(imf);
@@ -2507,7 +2632,7 @@ in6p_set_source_filters(struct inpcb *inp, struct sockopt *sopt)
 		goto out_im6f_rollback;
 
 	INP_WLOCK_ASSERT(inp);
-	IN6_MULTI_LOCK();
+	IN6_MULTI_LIST_LOCK();
 
 	/*
 	 * Begin state merge transaction at MLD layer.
@@ -2523,7 +2648,7 @@ in6p_set_source_filters(struct inpcb *inp, struct sockopt *sopt)
 			CTR1(KTR_MLD, "%s: failed mld downcall", __func__);
 	}
 
-	IN6_MULTI_UNLOCK();
+	IN6_MULTI_LIST_UNLOCK();
 
 out_im6f_rollback:
 	if (error)
@@ -2714,9 +2839,9 @@ sysctl_ip6_mcast_filters(SYSCTL_HANDLER_ARGS)
 		return (retval);
 
 	IN6_MULTI_LOCK();
-
+	IN6_MULTI_LIST_LOCK();
 	IF_ADDR_RLOCK(ifp);
-	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
+	CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
 		if (ifma->ifma_addr->sa_family != AF_INET6 ||
 		    ifma->ifma_protospec == NULL)
 			continue;
@@ -2746,6 +2871,7 @@ sysctl_ip6_mcast_filters(SYSCTL_HANDLER_ARGS)
 	}
 	IF_ADDR_RUNLOCK(ifp);
 
+	IN6_MULTI_LIST_UNLOCK();
 	IN6_MULTI_UNLOCK();
 
 	return (retval);