diff options
Diffstat (limited to 'mm/slub.c')
| -rw-r--r-- | mm/slub.c | 121 |
1 files changed, 50 insertions, 71 deletions
diff --git a/mm/slub.c b/mm/slub.c index 9ec774dc7009..0337e60db5ac 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2123,42 +2123,6 @@ static inline void init_slab_obj_exts(struct slab *slab) slab->obj_exts = 0; } -/* - * Calculate the allocation size for slabobj_ext array. - * - * When memory allocation profiling is enabled, the obj_exts array - * could be allocated from the same slab cache it's being allocated for. - * This would prevent the slab from ever being freed because it would - * always contain at least one allocated object (its own obj_exts array). - * - * To avoid this, increase the allocation size when we detect the array - * may come from the same cache, forcing it to use a different cache. - */ -static inline size_t obj_exts_alloc_size(struct kmem_cache *s, - struct slab *slab, gfp_t gfp) -{ - size_t sz = sizeof(struct slabobj_ext) * slab->objects; - struct kmem_cache *obj_exts_cache; - - if (sz > KMALLOC_MAX_CACHE_SIZE) - return sz; - - if (!is_kmalloc_normal(s)) - return sz; - - obj_exts_cache = kmalloc_slab(sz, NULL, gfp, __kmalloc_token(0)); - /* - * We can't simply compare s with obj_exts_cache, because partitioned kmalloc - * caches have multiple caches per size, selected by caller address or type. - * Since caller address or type may differ between kmalloc_slab() and actual - * allocation, bump size when sizes are equal. - */ - if (s->object_size == obj_exts_cache->object_size) - return obj_exts_cache->object_size + 1; - - return sz; -} - int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, gfp_t gfp, unsigned int alloc_flags) { @@ -2168,14 +2132,18 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, unsigned long new_exts; unsigned long old_exts; struct slabobj_ext *vec; - size_t sz; + size_t sz = sizeof(struct slabobj_ext) * slab->objects; gfp &= ~OBJCGS_CLEAR_MASK; - /* Prevent recursive extension vector allocation */ - alloc_flags |= SLAB_ALLOC_NO_RECURSE; - alloc_flags &= ~SLAB_ALLOC_NEW_SLAB; + /* + * In most cases, obj_exts arrays are allocated from normal kmalloc. + * However, normal kmalloc caches must allocate them from + * KMALLOC_NO_OBJ_EXT caches to prevent recursion. + */ + if (is_kmalloc_normal(s)) + alloc_flags |= SLAB_ALLOC_NO_OBJ_EXT; - sz = obj_exts_alloc_size(s, slab, gfp); + alloc_flags &= ~SLAB_ALLOC_NEW_SLAB; /* This will use kmalloc_nolock() if alloc_flags say so */ vec = kmalloc_flags(sz, gfp | __GFP_ZERO, alloc_flags, slab_nid(slab)); @@ -2193,8 +2161,21 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, return -ENOMEM; } - VM_WARN_ON_ONCE(virt_to_slab(vec) != NULL && - virt_to_slab(vec)->slab_cache == s); + if (IS_ENABLED(CONFIG_DEBUG_VM)) { + struct kmem_cache *exts_cache; + struct slab *exts_slab; + + exts_slab = virt_to_slab(vec); + if (exts_slab) { + /* + * The vector must be allocated from either normal or + * KMALLOC_NO_OBJ_EXT kmalloc caches to avoid cycles. + */ + exts_cache = exts_slab->slab_cache; + WARN_ON_ONCE(!is_kmalloc_normal(exts_cache) && + !(exts_cache->flags & SLAB_NO_OBJ_EXT)); + } + } new_exts = (unsigned long)vec; #ifdef CONFIG_MEMCG @@ -2217,7 +2198,6 @@ retry: * assign slabobj_exts in parallel. In this case the existing * objcg vector should be reused. */ - mark_obj_codetag_empty(vec); if (unlikely(!allow_spin)) kfree_nolock(vec); else @@ -2253,14 +2233,6 @@ static inline void free_slab_obj_exts(struct slab *slab, bool allow_spin) return; } - /* - * obj_exts was created with SLAB_ALLOC_NO_RECURSE flag, therefore its - * corresponding extension will be NULL. alloc_tag_sub() will throw a - * warning if slab has extensions but the extension of an object is - * NULL, therefore replace NULL with CODETAG_EMPTY to indicate that - * the extension for obj_exts is expected to be NULL. - */ - mark_obj_codetag_empty(obj_exts); if (allow_spin) kfree(obj_exts); else @@ -5356,7 +5328,7 @@ void *__do_kmalloc_node(kmem_buckets *b, gfp_t flags, int node, if (unlikely(!size)) return ZERO_SIZE_PTR; - s = kmalloc_slab(size, b, flags, token); + s = kmalloc_slab(size, b, flags, token, ac->alloc_flags); ret = slab_alloc_node(s, flags, node, ac); ret = kasan_kmalloc(s, ret, size, flags); @@ -5419,7 +5391,9 @@ static void *__kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_f retry: if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) return NULL; - s = kmalloc_slab(size, NULL, gfp_flags, PASS_TOKEN_PARAM(token)); + + s = kmalloc_slab(size, NULL, gfp_flags, PASS_TOKEN_PARAM(token), + ac->alloc_flags); if (!(s->flags & __CMPXCHG_DOUBLE) && !kmem_cache_debug(s)) /* @@ -6217,7 +6191,6 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) void *remote_objects[PCS_BATCH_MAX]; unsigned int remote_nr = 0; -next_remote_batch: while (i < size) { struct slab *slab = virt_to_slab(p[i]); @@ -6232,8 +6205,11 @@ next_remote_batch: if (unlikely(!can_free_to_pcs(slab))) { remote_objects[remote_nr] = p[i]; p[i] = p[--size]; - if (++remote_nr >= PCS_BATCH_MAX) - goto flush_remote; + if (++remote_nr >= PCS_BATCH_MAX) { + __kmem_cache_free_bulk(s, remote_nr, &remote_objects[0]); + stat_add(s, FREE_SLOWPATH, remote_nr); + remote_nr = 0; + } continue; } @@ -6317,10 +6293,6 @@ flush_remote: if (remote_nr) { __kmem_cache_free_bulk(s, remote_nr, &remote_objects[0]); stat_add(s, FREE_SLOWPATH, remote_nr); - if (i < size) { - remote_nr = 0; - goto next_remote_batch; - } } } @@ -7801,12 +7773,12 @@ static unsigned int calculate_sheaf_capacity(struct kmem_cache *s, return 0; /* - * Bootstrap caches can't have sheaves for now (SLAB_NO_OBJ_EXT). + * Bootstrap caches can't have sheaves for now (SLAB_NO_SHEAVES). * SLAB_NOLEAKTRACE caches (e.g., kmemleak's object_cache) must not * have sheaves to avoid recursion when sheaf allocation triggers * kmemleak tracking. */ - if (s->flags & (SLAB_NO_OBJ_EXT | SLAB_NOLEAKTRACE)) + if (s->flags & (SLAB_NO_SHEAVES | SLAB_NOLEAKTRACE)) return 0; /* @@ -7981,10 +7953,10 @@ static int calculate_sizes(struct kmem_cache_args *args, struct kmem_cache *s) s->allocflags |= __GFP_RECLAIMABLE; /* - * For KMALLOC_NORMAL caches we enable sheaves later by - * bootstrap_kmalloc_sheaves() to avoid recursion + * For kmalloc caches we enable sheaves later by + * bootstrap_kmalloc_sheaves() to avoid recursion. */ - if (!is_kmalloc_normal(s)) + if (!is_kmalloc_cache(s)) s->sheaf_capacity = calculate_sheaf_capacity(s, args); /* @@ -8499,6 +8471,8 @@ static void __init bootstrap_cache_sheaves(struct kmem_cache *s) bool failed = false; int node, cpu; + VM_WARN_ON_ONCE(cache_has_sheaves(s)); + capacity = calculate_sheaf_capacity(s, &empty_args); /* capacity can be 0 due to debugging or SLUB_TINY */ @@ -8548,10 +8522,13 @@ static void __init bootstrap_kmalloc_sheaves(void) { enum kmalloc_cache_type type; - for (type = KMALLOC_NORMAL; type <= KMALLOC_PARTITION_END; type++) { + for (type = KMALLOC_NORMAL; type < NR_KMALLOC_TYPES; type++) { for (int idx = 0; idx < KMALLOC_SHIFT_HIGH + 1; idx++) { - if (kmalloc_caches[type][idx]) - bootstrap_cache_sheaves(kmalloc_caches[type][idx]); + struct kmem_cache *s = kmalloc_caches[type][idx]; + + /* Do not bootstrap twice when caches are aliased */ + if (s && !cache_has_sheaves(s)) + bootstrap_cache_sheaves(s); } } } @@ -8583,7 +8560,8 @@ void __init kmem_cache_init(void) create_boot_cache(kmem_cache_node, "kmem_cache_node", sizeof(struct kmem_cache_node), - SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0); + SLAB_HWCACHE_ALIGN | SLAB_NO_SHEAVES | SLAB_NO_OBJ_EXT, + 0, 0); hotplug_node_notifier(slab_memory_callback, SLAB_CALLBACK_PRI); @@ -8593,7 +8571,8 @@ void __init kmem_cache_init(void) create_boot_cache(kmem_cache, "kmem_cache", offsetof(struct kmem_cache, per_node) + nr_node_ids * sizeof(struct kmem_cache_per_node_ptrs), - SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0); + SLAB_HWCACHE_ALIGN | SLAB_NO_SHEAVES | SLAB_NO_OBJ_EXT, + 0, 0); kmem_cache = bootstrap(&boot_kmem_cache); kmem_cache_node = bootstrap(&boot_kmem_cache_node); |