41 	struct blk_crypto_profile *profile;  member
44 static inline void blk_crypto_hw_enter(struct blk_crypto_profile *profile)  in blk_crypto_hw_enter()  argument
51 	if (profile->dev)  in blk_crypto_hw_enter()
52 		pm_runtime_get_sync(profile->dev);  in blk_crypto_hw_enter()
53 	down_write(&profile->lock);  in blk_crypto_hw_enter()
56 static inline void blk_crypto_hw_exit(struct blk_crypto_profile *profile)  in blk_crypto_hw_exit()  argument
58 	up_write(&profile->lock);  in blk_crypto_hw_exit()
59 	if (profile->dev)  in blk_crypto_hw_exit()
60 		pm_runtime_put_sync(profile->dev);  in blk_crypto_hw_exit()
73 int blk_crypto_profile_init(struct blk_crypto_profile *profile,  in blk_crypto_profile_init()  argument
80 	memset(profile, 0, sizeof(*profile));  in blk_crypto_profile_init()
81 	init_rwsem(&profile->lock);  in blk_crypto_profile_init()
88 	profile->slots = kvcalloc(num_slots, sizeof(profile->slots[0]),  in blk_crypto_profile_init()
90 	if (!profile->slots)  in blk_crypto_profile_init()
93 	profile->num_slots = num_slots;  in blk_crypto_profile_init()
95 	init_waitqueue_head(&profile->idle_slots_wait_queue);  in blk_crypto_profile_init()
96 	INIT_LIST_HEAD(&profile->idle_slots);  in blk_crypto_profile_init()
99 		profile->slots[slot].profile = profile;  in blk_crypto_profile_init()
100 		list_add_tail(&profile->slots[slot].idle_slot_node,  in blk_crypto_profile_init()
101 			      &profile->idle_slots);  in blk_crypto_profile_init()
104 	spin_lock_init(&profile->idle_slots_lock);  in blk_crypto_profile_init()
114 	profile->log_slot_ht_size = ilog2(slot_hashtable_size);  in blk_crypto_profile_init()
115 	profile->slot_hashtable =  in blk_crypto_profile_init()
117 			       sizeof(profile->slot_hashtable[0]), GFP_KERNEL);  in blk_crypto_profile_init()
118 	if (!profile->slot_hashtable)  in blk_crypto_profile_init()
121 		INIT_HLIST_HEAD(&profile->slot_hashtable[i]);  in blk_crypto_profile_init()
126 	blk_crypto_profile_destroy(profile);  in blk_crypto_profile_init()
131 static void blk_crypto_profile_destroy_callback(void *profile)  in blk_crypto_profile_destroy_callback()  argument
133 	blk_crypto_profile_destroy(profile);  in blk_crypto_profile_destroy_callback()
148 				 struct blk_crypto_profile *profile,  in devm_blk_crypto_profile_init()  argument
151 	int err = blk_crypto_profile_init(profile, num_slots);  in devm_blk_crypto_profile_init()
158 					profile);  in devm_blk_crypto_profile_init()
163 blk_crypto_hash_bucket_for_key(struct blk_crypto_profile *profile,  in blk_crypto_hash_bucket_for_key()  argument
166 	return &profile->slot_hashtable[  in blk_crypto_hash_bucket_for_key()
167 			hash_ptr(key, profile->log_slot_ht_size)];  in blk_crypto_hash_bucket_for_key()
173 	struct blk_crypto_profile *profile = slot->profile;  in blk_crypto_remove_slot_from_lru_list()  local
176 	spin_lock_irqsave(&profile->idle_slots_lock, flags);  in blk_crypto_remove_slot_from_lru_list()
178 	spin_unlock_irqrestore(&profile->idle_slots_lock, flags);  in blk_crypto_remove_slot_from_lru_list()
182 blk_crypto_find_keyslot(struct blk_crypto_profile *profile,  in blk_crypto_find_keyslot()  argument
186 		blk_crypto_hash_bucket_for_key(profile, key);  in blk_crypto_find_keyslot()
197 blk_crypto_find_and_grab_keyslot(struct blk_crypto_profile *profile,  in blk_crypto_find_and_grab_keyslot()  argument
202 	slot = blk_crypto_find_keyslot(profile, key);  in blk_crypto_find_and_grab_keyslot()
220 	return slot - slot->profile->slots;  in blk_crypto_keyslot_index()
241 blk_status_t blk_crypto_get_keyslot(struct blk_crypto_profile *profile,  in blk_crypto_get_keyslot()  argument
255 	if (profile->num_slots == 0)  in blk_crypto_get_keyslot()
258 	down_read(&profile->lock);  in blk_crypto_get_keyslot()
259 	slot = blk_crypto_find_and_grab_keyslot(profile, key);  in blk_crypto_get_keyslot()
260 	up_read(&profile->lock);  in blk_crypto_get_keyslot()
265 		blk_crypto_hw_enter(profile);  in blk_crypto_get_keyslot()
266 		slot = blk_crypto_find_and_grab_keyslot(profile, key);  in blk_crypto_get_keyslot()
268 			blk_crypto_hw_exit(profile);  in blk_crypto_get_keyslot()
276 		if (!list_empty(&profile->idle_slots))  in blk_crypto_get_keyslot()
279 		blk_crypto_hw_exit(profile);  in blk_crypto_get_keyslot()
280 		wait_event(profile->idle_slots_wait_queue,  in blk_crypto_get_keyslot()
281 			   !list_empty(&profile->idle_slots));  in blk_crypto_get_keyslot()
284 	slot = list_first_entry(&profile->idle_slots, struct blk_crypto_keyslot,  in blk_crypto_get_keyslot()
288 	err = profile->ll_ops.keyslot_program(profile, key, slot_idx);  in blk_crypto_get_keyslot()
290 		wake_up(&profile->idle_slots_wait_queue);  in blk_crypto_get_keyslot()
291 		blk_crypto_hw_exit(profile);  in blk_crypto_get_keyslot()
300 		       blk_crypto_hash_bucket_for_key(profile, key));  in blk_crypto_get_keyslot()
306 	blk_crypto_hw_exit(profile);  in blk_crypto_get_keyslot()
320 	struct blk_crypto_profile *profile;  in blk_crypto_put_keyslot()  local
326 	profile = slot->profile;  in blk_crypto_put_keyslot()
329 					&profile->idle_slots_lock, flags)) {  in blk_crypto_put_keyslot()
330 		list_add_tail(&slot->idle_slot_node, &profile->idle_slots);  in blk_crypto_put_keyslot()
331 		spin_unlock_irqrestore(&profile->idle_slots_lock, flags);  in blk_crypto_put_keyslot()
332 		wake_up(&profile->idle_slots_wait_queue);  in blk_crypto_put_keyslot()
344 bool __blk_crypto_cfg_supported(struct blk_crypto_profile *profile,  in __blk_crypto_cfg_supported()  argument
347 	if (!profile)  in __blk_crypto_cfg_supported()
349 	if (!(profile->modes_supported[cfg->crypto_mode] & cfg->data_unit_size))  in __blk_crypto_cfg_supported()
351 	if (profile->max_dun_bytes_supported < cfg->dun_bytes)  in __blk_crypto_cfg_supported()
373 int __blk_crypto_evict_key(struct blk_crypto_profile *profile,  in __blk_crypto_evict_key()  argument
379 	if (profile->num_slots == 0) {  in __blk_crypto_evict_key()
380 		if (profile->ll_ops.keyslot_evict) {  in __blk_crypto_evict_key()
381 			blk_crypto_hw_enter(profile);  in __blk_crypto_evict_key()
382 			err = profile->ll_ops.keyslot_evict(profile, key, -1);  in __blk_crypto_evict_key()
383 			blk_crypto_hw_exit(profile);  in __blk_crypto_evict_key()
389 	blk_crypto_hw_enter(profile);  in __blk_crypto_evict_key()
390 	slot = blk_crypto_find_keyslot(profile, key);  in __blk_crypto_evict_key()
398 	err = profile->ll_ops.keyslot_evict(profile, key,  in __blk_crypto_evict_key()
407 	blk_crypto_hw_exit(profile);  in __blk_crypto_evict_key()
420 void blk_crypto_reprogram_all_keys(struct blk_crypto_profile *profile)  in blk_crypto_reprogram_all_keys()  argument
424 	if (profile->num_slots == 0)  in blk_crypto_reprogram_all_keys()
428 	down_write(&profile->lock);  in blk_crypto_reprogram_all_keys()
429 	for (slot = 0; slot < profile->num_slots; slot++) {  in blk_crypto_reprogram_all_keys()
430 		const struct blk_crypto_key *key = profile->slots[slot].key;  in blk_crypto_reprogram_all_keys()
436 		err = profile->ll_ops.keyslot_program(profile, key, slot);  in blk_crypto_reprogram_all_keys()
439 	up_write(&profile->lock);  in blk_crypto_reprogram_all_keys()
443 void blk_crypto_profile_destroy(struct blk_crypto_profile *profile)  in blk_crypto_profile_destroy()  argument
445 	if (!profile)  in blk_crypto_profile_destroy()
447 	kvfree(profile->slot_hashtable);  in blk_crypto_profile_destroy()
448 	kvfree_sensitive(profile->slots,  in blk_crypto_profile_destroy()
449 			 sizeof(profile->slots[0]) * profile->num_slots);  in blk_crypto_profile_destroy()
450 	memzero_explicit(profile, sizeof(*profile));  in blk_crypto_profile_destroy()
454 bool blk_crypto_register(struct blk_crypto_profile *profile,  in blk_crypto_register()  argument
461 	q->crypto_profile = profile;  in blk_crypto_register()