ANDROID: mm/slab: Create 64-byte slab caches if the hardware supports it

Currently, ARM64 hardcodes ARCH_DMA_MINALIGN to 128 bytes, which
ultimately causes the smallest kmalloc cache in the system to allocate
128 byte objects. This is wasteful on systems that don't have this 128
byte minimum DMA alignment requirement, as their cachelines are 64 bytes
in size, and thus, can function properly with kmalloc allocating objects
64 bytes in size.

Add a commandline parameter--android_kmalloc_64_create--to create and use
all of the different types of kmalloc caches (e.g. kmalloc-*,
kmalloc-rcl-*, etc) for 64-byte allocations, if the hardware supports it.
The hardware supports 64 byte objects if their size is aligned to the cache
line size that is reported by the system.

This patch is loosely based on the discussion in [1] and aims to
optimize the amount of memory kmalloc uses by allowing slab allocations
to be satisfied by the kmalloc-64 cache, if the hardware supports it.

This approach differs from [1] by being less invasive, and only attempting
to create the kmalloc-64 cache if the kmalloc_64_create parameter is
specified in the kernel commandline. The intent of this is to preserve
the ABI semantics of kmalloc (i.e. 128 or ARCH_DMA_MINALIGN byte alignment)
for existing devices using ARM64 GKI kernels on the android13-5.15 branch,
since the KMI is frozen.

The final solution from [1] should be used, or this patch should be
reworked for future versions of Android and should not be cherry-picked
forward.

[1] https://lore.kernel.org/linux-mm/20220405135758.774016-1-catalin.marinas@arm.com/

Bug: 241844128
Signed-off-by: Isaac J. Manjarres <isaacmanjarres@google.com>
Change-Id: I6d39e6c6957c944f0327e6774afd33f4c2230019

Documentation/admin-guide/kernel-parameters.txt

@@ -338,6 +338,10 @@
 			connected to one of 16 gameports
 			Format: <type1>,<type2>,..<type16>
 
+	android_kmalloc_64_create [MM]
+			Creates all kmalloc variants of the 64-byte slab cache,
+			if the hardware supports it.
+
 	apc=		[HW,SPARC]
 			Power management functions (SPARCstation-4/5 + deriv.)
 			Format: noidle

include/linux/slab.h

@@ -343,6 +343,8 @@ enum kmalloc_cache_type {
 extern struct kmem_cache *
 kmalloc_caches[NR_KMALLOC_TYPES][KMALLOC_SHIFT_HIGH + 1];
 
+extern bool android_kmalloc_64_create;
+
 /*
  * Define gfp bits that should not be set for KMALLOC_NORMAL.
  */
@@ -394,6 +396,9 @@ static __always_inline unsigned int __kmalloc_index(size_t size,
 	if (!size)
 		return 0;
 
+	if (android_kmalloc_64_create && size <= 64)
+		return 6;
+
 	if (size <= KMALLOC_MIN_SIZE)
 		return KMALLOC_SHIFT_LOW;
 

mm/slab_common.c

@@ -646,6 +646,15 @@ EXPORT_SYMBOL_GPL(kmem_dump_obj);
 #endif
 
 #ifndef CONFIG_SLOB
+static int __init setup_android_kmalloc_64_create(char *str)
+{
+	if (IS_ALIGNED(64, cache_line_size()))
+		android_kmalloc_64_create = true;
+
+	return 1;
+}
+__setup("android_kmalloc_64_create", setup_android_kmalloc_64_create);
+
 /* Create a cache during boot when no slab services are available yet */
 void __init create_boot_cache(struct kmem_cache *s, const char *name,
 		unsigned int size, slab_flags_t flags,
@@ -654,6 +663,14 @@ void __init create_boot_cache(struct kmem_cache *s, const char *name,
 	int err;
 	unsigned int align = ARCH_KMALLOC_MINALIGN;
 
+	/*
+	 * Ensure object alignment is 64. Otherwise, it can be larger
+	 * (e.g. 128 with ARM64), which causes SLUB to increase the object
+	 * size to 128 bytes to conform with the alignment.
+	 */
+	if (android_kmalloc_64_create && size == 64)
+		align = 64;
+
 	s->name = name;
 	s->size = s->object_size = size;
 
@@ -698,6 +715,9 @@ kmalloc_caches[NR_KMALLOC_TYPES][KMALLOC_SHIFT_HIGH + 1] __ro_after_init =
 { /* initialization for https://bugs.llvm.org/show_bug.cgi?id=42570 */ };
 EXPORT_SYMBOL(kmalloc_caches);
 
+bool android_kmalloc_64_create __ro_after_init;
+EXPORT_SYMBOL_GPL(android_kmalloc_64_create);
+
 /*
  * Conversion table for small slabs sizes / 8 to the index in the
  * kmalloc array. This is necessary for slabs < 192 since we have non power
@@ -844,6 +864,10 @@ void __init setup_kmalloc_cache_index_table(void)
 		size_index[elem] = KMALLOC_SHIFT_LOW;
 	}
 
+	if (android_kmalloc_64_create)
+		for (i = 8; i <= 64; i += 8)
+			size_index[size_index_elem(i)] = 6;
+
 	if (KMALLOC_MIN_SIZE >= 64) {
 		/*
 		 * The 96 byte size cache is not used if the alignment
@@ -901,6 +925,10 @@ void __init create_kmalloc_caches(slab_flags_t flags)
 	int i;
 	enum kmalloc_cache_type type;
 
+	if (android_kmalloc_64_create)
+		for (type = KMALLOC_NORMAL; type <= KMALLOC_RECLAIM; type++)
+			new_kmalloc_cache(6, type, flags);
+
 	/*
 	 * Including KMALLOC_CGROUP if CONFIG_MEMCG_KMEM defined
 	 */