diff options
Diffstat (limited to 'freebsd/sys/vm/uma_int.h')
| -rw-r--r-- | freebsd/sys/vm/uma_int.h | 84 |
1 files changed, 42 insertions, 42 deletions
diff --git a/freebsd/sys/vm/uma_int.h b/freebsd/sys/vm/uma_int.h index 0429bac6..1d055ed6 100644 --- a/freebsd/sys/vm/uma_int.h +++ b/freebsd/sys/vm/uma_int.h @@ -30,6 +30,7 @@ * */ +#include <sys/counter.h> #include <sys/_bitset.h> #include <sys/_domainset.h> #include <sys/_task.h> @@ -178,8 +179,8 @@ SLIST_HEAD(slabhead, uma_slab); struct uma_hash { struct slabhead *uh_slab_hash; /* Hash table for slabs */ - int uh_hashsize; /* Current size of the hash table */ - int uh_hashmask; /* Mask used during hashing */ + u_int uh_hashsize; /* Current size of the hash table */ + u_int uh_hashmask; /* Mask used during hashing */ }; /* @@ -196,7 +197,7 @@ struct uma_hash { */ struct uma_bucket { - LIST_ENTRY(uma_bucket) ub_link; /* Link into the zone */ + TAILQ_ENTRY(uma_bucket) ub_link; /* Link into the zone */ int16_t ub_cnt; /* Count of items in bucket. */ int16_t ub_entries; /* Max items. */ void *ub_bucket[]; /* actual allocation storage */ @@ -207,6 +208,7 @@ typedef struct uma_bucket * uma_bucket_t; struct uma_cache { uma_bucket_t uc_freebucket; /* Bucket we're freeing to */ uma_bucket_t uc_allocbucket; /* Bucket to allocate from */ + uma_bucket_t uc_crossbucket; /* cross domain bucket */ uint64_t uc_allocs; /* Count of allocations */ uint64_t uc_frees; /* Count of frees */ } UMA_ALIGN; @@ -231,7 +233,9 @@ typedef struct uma_domain * uma_domain_t; * */ struct uma_keg { - struct mtx uk_lock; /* Lock for the keg */ + struct mtx uk_lock; /* Lock for the keg must be first. + * See shared uz_keg/uz_lockptr + * member of struct uma_zone. */ struct uma_hash uk_hash; LIST_HEAD(,uma_zone) uk_zones; /* Keg's zones */ @@ -244,7 +248,6 @@ struct uma_keg { uint32_t uk_reserve; /* Number of reserved items. */ uint32_t uk_size; /* Requested size of each item */ uint32_t uk_rsize; /* Real size of each item */ - uint32_t uk_maxpages; /* Maximum number of pages to alloc */ uma_init uk_init; /* Keg's init routine */ uma_fini uk_fini; /* Keg's fini routine */ @@ -308,16 +311,11 @@ struct uma_slab { #endif typedef struct uma_slab * uma_slab_t; -typedef uma_slab_t (*uma_slaballoc)(uma_zone_t, uma_keg_t, int, int); -struct uma_klink { - LIST_ENTRY(uma_klink) kl_link; - uma_keg_t kl_keg; -}; -typedef struct uma_klink *uma_klink_t; +TAILQ_HEAD(uma_bucketlist, uma_bucket); struct uma_zone_domain { - LIST_HEAD(,uma_bucket) uzd_buckets; /* full buckets */ + struct uma_bucketlist uzd_buckets; /* full buckets */ long uzd_nitems; /* total item count */ long uzd_imax; /* maximum item count this period */ long uzd_imin; /* minimum item count this period */ @@ -334,8 +332,10 @@ typedef struct uma_zone_domain * uma_zone_domain_t; */ struct uma_zone { /* Offset 0, used in alloc/free fast/medium fast path and const. */ - struct mtx *uz_lockptr; - const char *uz_name; /* Text name of the zone */ + union { + uma_keg_t uz_keg; /* This zone's keg */ + struct mtx *uz_lockptr; /* To keg or to self */ + }; #ifndef __rtems__ struct uma_zone_domain *uz_domain; /* per-domain buckets */ #else /* __rtems__ */ @@ -345,19 +345,21 @@ struct uma_zone { uint32_t uz_size; /* Size inherited from kegs */ uma_ctor uz_ctor; /* Constructor for each allocation */ uma_dtor uz_dtor; /* Destructor */ - uma_init uz_init; /* Initializer for each item */ - uma_fini uz_fini; /* Finalizer for each item. */ + uint64_t uz_items; /* Total items count */ + uint64_t uz_max_items; /* Maximum number of items to alloc */ + uint32_t uz_sleepers; /* Number of sleepers on memory */ + uint16_t uz_count; /* Amount of items in full bucket */ + uint16_t uz_count_max; /* Maximum amount of items there */ /* Offset 64, used in bucket replenish. */ uma_import uz_import; /* Import new memory to cache. */ uma_release uz_release; /* Release memory from cache. */ void *uz_arg; /* Import/release argument. */ - uma_slaballoc uz_slab; /* Allocate a slab from the backend. */ - uint16_t uz_count; /* Amount of items in full bucket */ - uint16_t uz_count_min; /* Minimal amount of items there */ - /* 32bit pad on 64bit. */ - LIST_ENTRY(uma_zone) uz_link; /* List of all zones in keg */ - LIST_HEAD(,uma_klink) uz_kegs; /* List of kegs. */ + uma_init uz_init; /* Initializer for each item */ + uma_fini uz_fini; /* Finalizer for each item. */ + void *uz_spare; + uint64_t uz_bkt_count; /* Items in bucket cache */ + uint64_t uz_bkt_max; /* Maximum bucket cache size */ /* Offset 128 Rare. */ /* @@ -366,19 +368,20 @@ struct uma_zone { * members to reduce alignment overhead. */ struct mtx uz_lock; /* Lock for the zone */ - struct uma_klink uz_klink; /* klink for first keg. */ + LIST_ENTRY(uma_zone) uz_link; /* List of all zones in keg */ + const char *uz_name; /* Text name of the zone */ /* The next two fields are used to print a rate-limited warnings. */ const char *uz_warning; /* Warning to print on failure */ struct timeval uz_ratecheck; /* Warnings rate-limiting */ struct task uz_maxaction; /* Task to run when at limit */ + uint16_t uz_count_min; /* Minimal amount of items in bucket */ - /* 16 bytes of pad. */ - - /* Offset 256, atomic stats. */ - volatile u_long uz_allocs UMA_ALIGN; /* Total number of allocations */ - volatile u_long uz_fails; /* Total number of alloc failures */ - volatile u_long uz_frees; /* Total number of frees */ + /* Offset 256, stats. */ + counter_u64_t uz_allocs; /* Total number of allocations */ + counter_u64_t uz_frees; /* Total number of frees */ + counter_u64_t uz_fails; /* Total number of alloc failures */ uint64_t uz_sleeps; /* Total number of alloc sleeps */ + uint64_t uz_xdomain; /* Total number of cross-domain frees */ /* * This HAS to be the last item because we adjust the zone size @@ -392,25 +395,15 @@ struct uma_zone { /* * These flags must not overlap with the UMA_ZONE flags specified in uma.h. */ -#define UMA_ZFLAG_MULTI 0x04000000 /* Multiple kegs in the zone. */ -#define UMA_ZFLAG_DRAINING 0x08000000 /* Running zone_drain. */ +#define UMA_ZFLAG_CACHE 0x04000000 /* uma_zcache_create()d it */ +#define UMA_ZFLAG_RECLAIMING 0x08000000 /* Running zone_reclaim(). */ #define UMA_ZFLAG_BUCKET 0x10000000 /* Bucket zone. */ #define UMA_ZFLAG_INTERNAL 0x20000000 /* No offpage no PCPU. */ -#define UMA_ZFLAG_FULL 0x40000000 /* Reached uz_maxpages */ #define UMA_ZFLAG_CACHEONLY 0x80000000 /* Don't ask VM for buckets. */ #define UMA_ZFLAG_INHERIT \ (UMA_ZFLAG_INTERNAL | UMA_ZFLAG_CACHEONLY | UMA_ZFLAG_BUCKET) -static inline uma_keg_t -zone_first_keg(uma_zone_t zone) -{ - uma_klink_t klink; - - klink = LIST_FIRST(&zone->uz_kegs); - return (klink != NULL) ? klink->kl_keg : NULL; -} - #undef UMA_ALIGN #ifdef _KERNEL @@ -435,6 +428,13 @@ void uma_large_free(uma_slab_t slab); #define KEG_LOCK_FINI(k) mtx_destroy(&(k)->uk_lock) #define KEG_LOCK(k) mtx_lock(&(k)->uk_lock) #define KEG_UNLOCK(k) mtx_unlock(&(k)->uk_lock) +#define KEG_LOCK_ASSERT(k) mtx_assert(&(k)->uk_lock, MA_OWNED) + +#define KEG_GET(zone, keg) do { \ + (keg) = (zone)->uz_keg; \ + KASSERT((void *)(keg) != (void *)&(zone)->uz_lock, \ + ("%s: Invalid zone %p type", __func__, (zone))); \ + } while (0) #define ZONE_LOCK_INIT(z, lc) \ do { \ @@ -467,7 +467,7 @@ static __inline uma_slab_t hash_sfind(struct uma_hash *hash, uint8_t *data) { uma_slab_t slab; - int hval; + u_int hval; hval = UMA_HASH(hash, data); |