78 		struct bpf_cgroup_storage *storage;  in cgroup_storage_lookup()  local
80 		storage = container_of(node, struct bpf_cgroup_storage, node);  in cgroup_storage_lookup()
82 		switch (bpf_cgroup_storage_key_cmp(map, key, &storage->key)) {  in cgroup_storage_lookup()
92 			return storage;  in cgroup_storage_lookup()
103 				 struct bpf_cgroup_storage *storage)  in cgroup_storage_insert()  argument
114 		switch (bpf_cgroup_storage_key_cmp(map, &storage->key, &this->key)) {  in cgroup_storage_insert()
126 	rb_link_node(&storage->node, parent, new);  in cgroup_storage_insert()
127 	rb_insert_color(&storage->node, root);  in cgroup_storage_insert()
135 	struct bpf_cgroup_storage *storage;  in cgroup_storage_lookup_elem()  local
137 	storage = cgroup_storage_lookup(map, key, false);  in cgroup_storage_lookup_elem()
138 	if (!storage)  in cgroup_storage_lookup_elem()
141 	return &READ_ONCE(storage->buf)->data[0];  in cgroup_storage_lookup_elem()
147 	struct bpf_cgroup_storage *storage;  in cgroup_storage_update_elem()  local
157 	storage = cgroup_storage_lookup((struct bpf_cgroup_storage_map *)map,  in cgroup_storage_update_elem()
159 	if (!storage)  in cgroup_storage_update_elem()
163 		copy_map_value_locked(map, storage->buf->data, value, false);  in cgroup_storage_update_elem()
176 	new = xchg(&storage->buf, new);  in cgroup_storage_update_elem()
186 	struct bpf_cgroup_storage *storage;  in bpf_percpu_cgroup_storage_copy()  local
191 	storage = cgroup_storage_lookup(map, key, false);  in bpf_percpu_cgroup_storage_copy()
192 	if (!storage) {  in bpf_percpu_cgroup_storage_copy()
204 				per_cpu_ptr(storage->percpu_buf, cpu), size);  in bpf_percpu_cgroup_storage_copy()
215 	struct bpf_cgroup_storage *storage;  in bpf_percpu_cgroup_storage_update()  local
223 	storage = cgroup_storage_lookup(map, key, false);  in bpf_percpu_cgroup_storage_update()
224 	if (!storage) {  in bpf_percpu_cgroup_storage_update()
237 		bpf_long_memcpy(per_cpu_ptr(storage->percpu_buf, cpu),  in bpf_percpu_cgroup_storage_update()
249 	struct bpf_cgroup_storage *storage;  in cgroup_storage_get_next_key()  local
257 		storage = cgroup_storage_lookup(map, key, true);  in cgroup_storage_get_next_key()
258 		if (!storage)  in cgroup_storage_get_next_key()
261 		storage = list_next_entry(storage, list_map);  in cgroup_storage_get_next_key()
262 		if (!storage)  in cgroup_storage_get_next_key()
265 		storage = list_first_entry(&map->list,  in cgroup_storage_get_next_key()
273 		*next = storage->key;  in cgroup_storage_get_next_key()
276 		*next = storage->key.cgroup_inode_id;  in cgroup_storage_get_next_key()
334 	struct bpf_cgroup_storage *storage, *stmp;  in cgroup_storage_map_free()  local
338 	list_for_each_entry_safe(storage, stmp, storages, list_map) {  in cgroup_storage_map_free()
339 		bpf_cgroup_storage_unlink(storage);  in cgroup_storage_map_free()
340 		bpf_cgroup_storage_free(storage);  in cgroup_storage_map_free()
418 	struct bpf_cgroup_storage *storage;  in cgroup_storage_seq_show_elem()  local
422 	storage = cgroup_storage_lookup(map_to_storage(map), key, false);  in cgroup_storage_seq_show_elem()
423 	if (!storage) {  in cgroup_storage_seq_show_elem()
433 				  &READ_ONCE(storage->buf)->data[0], m);  in cgroup_storage_seq_show_elem()
440 					  per_cpu_ptr(storage->percpu_buf, cpu),  in cgroup_storage_seq_show_elem()
496 	struct bpf_cgroup_storage *storage;  in bpf_cgroup_storage_alloc()  local
507 	storage = bpf_map_kmalloc_node(map, sizeof(struct bpf_cgroup_storage),  in bpf_cgroup_storage_alloc()
509 	if (!storage)  in bpf_cgroup_storage_alloc()
513 		storage->buf = bpf_map_kmalloc_node(map, size, gfp,  in bpf_cgroup_storage_alloc()
515 		if (!storage->buf)  in bpf_cgroup_storage_alloc()
517 		check_and_init_map_value(map, storage->buf->data);  in bpf_cgroup_storage_alloc()
519 		storage->percpu_buf = bpf_map_alloc_percpu(map, size, 8, gfp);  in bpf_cgroup_storage_alloc()
520 		if (!storage->percpu_buf)  in bpf_cgroup_storage_alloc()
524 	storage->map = (struct bpf_cgroup_storage_map *)map;  in bpf_cgroup_storage_alloc()
526 	return storage;  in bpf_cgroup_storage_alloc()
529 	kfree(storage);  in bpf_cgroup_storage_alloc()
535 	struct bpf_cgroup_storage *storage =  in free_shared_cgroup_storage_rcu()  local
538 	kfree(storage->buf);  in free_shared_cgroup_storage_rcu()
539 	kfree(storage);  in free_shared_cgroup_storage_rcu()
544 	struct bpf_cgroup_storage *storage =  in free_percpu_cgroup_storage_rcu()  local
547 	free_percpu(storage->percpu_buf);  in free_percpu_cgroup_storage_rcu()
548 	kfree(storage);  in free_percpu_cgroup_storage_rcu()
551 void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage)  in bpf_cgroup_storage_free()  argument
556 	if (!storage)  in bpf_cgroup_storage_free()
559 	map = &storage->map->map;  in bpf_cgroup_storage_free()
562 		call_rcu(&storage->rcu, free_shared_cgroup_storage_rcu);  in bpf_cgroup_storage_free()
564 		call_rcu(&storage->rcu, free_percpu_cgroup_storage_rcu);  in bpf_cgroup_storage_free()
567 void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage,  in bpf_cgroup_storage_link()  argument
573 	if (!storage)  in bpf_cgroup_storage_link()
576 	storage->key.attach_type = type;  in bpf_cgroup_storage_link()
577 	storage->key.cgroup_inode_id = cgroup_id(cgroup);  in bpf_cgroup_storage_link()
579 	map = storage->map;  in bpf_cgroup_storage_link()
582 	WARN_ON(cgroup_storage_insert(map, storage));  in bpf_cgroup_storage_link()
583 	list_add(&storage->list_map, &map->list);  in bpf_cgroup_storage_link()
584 	list_add(&storage->list_cg, &cgroup->bpf.storages);  in bpf_cgroup_storage_link()
588 void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage)  in bpf_cgroup_storage_unlink()  argument
593 	if (!storage)  in bpf_cgroup_storage_unlink()
596 	map = storage->map;  in bpf_cgroup_storage_unlink()
600 	rb_erase(&storage->node, root);  in bpf_cgroup_storage_unlink()
602 	list_del(&storage->list_map);  in bpf_cgroup_storage_unlink()
603 	list_del(&storage->list_cg);  in bpf_cgroup_storage_unlink()