163 #define slab_bufctl(slabp) \  argument
164 	((kmem_bufctl_t *)(((slab_t*)slabp)+1))
555 static void kmem_slab_destroy (kmem_cache_t *cachep, slab_t *slabp)  in kmem_slab_destroy()  argument
564 			void* objp = slabp->s_mem+cachep->objsize*i;  in kmem_slab_destroy()
588 	kmem_freepages(cachep, slabp->s_mem-slabp->colouroff);  in kmem_slab_destroy()
590 		kmem_cache_free(cachep->slabp_cache, slabp);  in kmem_slab_destroy()
919 	slab_t *slabp;  in __kmem_cache_shrink_locked()  local
930 		slabp = list_entry(cachep->slabs_free.prev, slab_t, list);  in __kmem_cache_shrink_locked()
932 		if (slabp->inuse)  in __kmem_cache_shrink_locked()
935 		list_del(&slabp->list);  in __kmem_cache_shrink_locked()
938 		kmem_slab_destroy(cachep, slabp);  in __kmem_cache_shrink_locked()
1035 	slab_t *slabp;  in kmem_cache_slabmgmt()  local
1039 		slabp = kmem_cache_alloc(cachep->slabp_cache, local_flags);  in kmem_cache_slabmgmt()
1040 		if (!slabp)  in kmem_cache_slabmgmt()
1047 		slabp = objp+colour_off;  in kmem_cache_slabmgmt()
1051 	slabp->inuse = 0;  in kmem_cache_slabmgmt()
1052 	slabp->colouroff = colour_off;  in kmem_cache_slabmgmt()
1053 	slabp->s_mem = objp+colour_off;  in kmem_cache_slabmgmt()
1055 	return slabp;  in kmem_cache_slabmgmt()
1059 			slab_t * slabp, unsigned long ctor_flags)  in kmem_cache_init_objs()  argument
1064 		void* objp = slabp->s_mem+cachep->objsize*i;  in kmem_cache_init_objs()
1095 		slab_bufctl(slabp)[i] = i+1;  in kmem_cache_init_objs()  local
1097 	slab_bufctl(slabp)[i-1] = BUFCTL_END;  in kmem_cache_init_objs()  local
1098 	slabp->free = 0;  in kmem_cache_init_objs()
1107 	slab_t	*slabp;  in kmem_cache_grow()  local
1169 	if (!(slabp = kmem_cache_slabmgmt(cachep, objp, offset, local_flags)))  in kmem_cache_grow()
1177 		SET_PAGE_SLAB(page, slabp);  in kmem_cache_grow()
1182 	kmem_cache_init_objs(cachep, slabp, ctor_flags);  in kmem_cache_grow()
1188 	list_add_tail(&slabp->list, &cachep->slabs_free);  in kmem_cache_grow()
1212 			slab_t *slabp, void * objp)  in kmem_extra_free_checks()  argument
1215 	unsigned int objnr = (objp-slabp->s_mem)/cachep->objsize;  in kmem_extra_free_checks()
1219 	if (objp != slabp->s_mem + objnr*cachep->objsize)  in kmem_extra_free_checks()
1223 	for (i = slabp->free; i != BUFCTL_END; i = slab_bufctl(slabp)[i]) {  in kmem_extra_free_checks()
1243 						slab_t *slabp)  in kmem_cache_alloc_one_tail()  argument
1252 	slabp->inuse++;  in kmem_cache_alloc_one_tail()
1253 	objp = slabp->s_mem + slabp->free*cachep->objsize;  in kmem_cache_alloc_one_tail()
1254 	slabp->free=slab_bufctl(slabp)[slabp->free];  in kmem_cache_alloc_one_tail()
1256 	if (unlikely(slabp->free == BUFCTL_END)) {  in kmem_cache_alloc_one_tail()
1257 		list_del(&slabp->list);  in kmem_cache_alloc_one_tail()
1258 		list_add(&slabp->list, &cachep->slabs_full);  in kmem_cache_alloc_one_tail()
1286 	slab_t *slabp;						\
1300 	slabp = list_entry(entry, slab_t, list);		\
1301 	kmem_cache_alloc_one_tail(cachep, slabp);		\
1312 		slab_t *slabp;  in kmem_cache_alloc_batch()  local
1326 		slabp = list_entry(entry, slab_t, list);  in kmem_cache_alloc_batch()
1328 				kmem_cache_alloc_one_tail(cachep, slabp);  in kmem_cache_alloc_batch()
1416 	slab_t* slabp;  in kmem_cache_free_one()  local
1425 	slabp = GET_PAGE_SLAB(virt_to_page(objp));  in kmem_cache_free_one()
1447 	if (kmem_extra_free_checks(cachep, slabp, objp))  in kmem_cache_free_one()
1451 		unsigned int objnr = (objp-slabp->s_mem)/cachep->objsize;  in kmem_cache_free_one()
1453 		slab_bufctl(slabp)[objnr] = slabp->free;  in kmem_cache_free_one()  local
1454 		slabp->free = objnr;  in kmem_cache_free_one()
1460 		int inuse = slabp->inuse;  in kmem_cache_free_one()
1461 		if (unlikely(!--slabp->inuse)) {  in kmem_cache_free_one()
1463 			list_del(&slabp->list);  in kmem_cache_free_one()
1464 			list_add(&slabp->list, &cachep->slabs_free);  in kmem_cache_free_one()
1467 			list_del(&slabp->list);  in kmem_cache_free_one()
1468 			list_add(&slabp->list, &cachep->slabs_partial);  in kmem_cache_free_one()
1740 	slab_t *slabp;  in kmem_cache_reap()  local
1788 			slabp = list_entry(p, slab_t, list);  in kmem_cache_reap()
1790 			if (slabp->inuse)  in kmem_cache_reap()
1841 		slabp = list_entry(p,slab_t,list);  in kmem_cache_reap()
1843 		if (slabp->inuse)  in kmem_cache_reap()
1846 		list_del(&slabp->list);  in kmem_cache_reap()
1853 		kmem_slab_destroy(best_cachep, slabp);  in kmem_cache_reap()
1901 	slab_t		*slabp;  in s_show()  local
1928 		slabp = list_entry(q, slab_t, list);  in s_show()
1929 		if (slabp->inuse != cachep->num)  in s_show()
1935 		slabp = list_entry(q, slab_t, list);  in s_show()
1936 		if (slabp->inuse == cachep->num || !slabp->inuse)  in s_show()
1938 		active_objs += slabp->inuse;  in s_show()
1942 		slabp = list_entry(q, slab_t, list);  in s_show()
1943 		if (slabp->inuse)  in s_show()