43 	struct blk_keyslot_manager *ksm;  member
46 static inline void blk_ksm_hw_enter(struct blk_keyslot_manager *ksm)  in blk_ksm_hw_enter()  argument
53 	if (ksm->dev)  in blk_ksm_hw_enter()
54 		pm_runtime_get_sync(ksm->dev);  in blk_ksm_hw_enter()
55 	down_write(&ksm->lock);  in blk_ksm_hw_enter()
58 static inline void blk_ksm_hw_exit(struct blk_keyslot_manager *ksm)  in blk_ksm_hw_exit()  argument
60 	up_write(&ksm->lock);  in blk_ksm_hw_exit()
61 	if (ksm->dev)  in blk_ksm_hw_exit()
62 		pm_runtime_put_sync(ksm->dev);  in blk_ksm_hw_exit()
75 int blk_ksm_init(struct blk_keyslot_manager *ksm, unsigned int num_slots)  in blk_ksm_init()  argument
81 	memset(ksm, 0, sizeof(*ksm));  in blk_ksm_init()
86 	ksm->slots = kvcalloc(num_slots, sizeof(ksm->slots[0]), GFP_KERNEL);  in blk_ksm_init()
87 	if (!ksm->slots)  in blk_ksm_init()
90 	ksm->num_slots = num_slots;  in blk_ksm_init()
92 	init_rwsem(&ksm->lock);  in blk_ksm_init()
94 	init_waitqueue_head(&ksm->idle_slots_wait_queue);  in blk_ksm_init()
95 	INIT_LIST_HEAD(&ksm->idle_slots);  in blk_ksm_init()
98 		ksm->slots[slot].ksm = ksm;  in blk_ksm_init()
99 		list_add_tail(&ksm->slots[slot].idle_slot_node,  in blk_ksm_init()
100 			      &ksm->idle_slots);  in blk_ksm_init()
103 	spin_lock_init(&ksm->idle_slots_lock);  in blk_ksm_init()
113 	ksm->log_slot_ht_size = ilog2(slot_hashtable_size);  in blk_ksm_init()
114 	ksm->slot_hashtable = kvmalloc_array(slot_hashtable_size,  in blk_ksm_init()
115 					     sizeof(ksm->slot_hashtable[0]),  in blk_ksm_init()
117 	if (!ksm->slot_hashtable)  in blk_ksm_init()
120 		INIT_HLIST_HEAD(&ksm->slot_hashtable[i]);  in blk_ksm_init()
125 	blk_ksm_destroy(ksm);  in blk_ksm_init()
131 blk_ksm_hash_bucket_for_key(struct blk_keyslot_manager *ksm,  in blk_ksm_hash_bucket_for_key()  argument
134 	return &ksm->slot_hashtable[hash_ptr(key, ksm->log_slot_ht_size)];  in blk_ksm_hash_bucket_for_key()
139 	struct blk_keyslot_manager *ksm = slot->ksm;  in blk_ksm_remove_slot_from_lru_list()  local
142 	spin_lock_irqsave(&ksm->idle_slots_lock, flags);  in blk_ksm_remove_slot_from_lru_list()
144 	spin_unlock_irqrestore(&ksm->idle_slots_lock, flags);  in blk_ksm_remove_slot_from_lru_list()
148 					struct blk_keyslot_manager *ksm,  in blk_ksm_find_keyslot()  argument
151 	const struct hlist_head *head = blk_ksm_hash_bucket_for_key(ksm, key);  in blk_ksm_find_keyslot()
162 					struct blk_keyslot_manager *ksm,  in blk_ksm_find_and_grab_keyslot()  argument
167 	slot = blk_ksm_find_keyslot(ksm, key);  in blk_ksm_find_and_grab_keyslot()
179 	return slot - slot->ksm->slots;  in blk_ksm_get_slot_idx()
199 blk_status_t blk_ksm_get_slot_for_key(struct blk_keyslot_manager *ksm,  in blk_ksm_get_slot_for_key()  argument
208 	down_read(&ksm->lock);  in blk_ksm_get_slot_for_key()
209 	slot = blk_ksm_find_and_grab_keyslot(ksm, key);  in blk_ksm_get_slot_for_key()
210 	up_read(&ksm->lock);  in blk_ksm_get_slot_for_key()
215 		blk_ksm_hw_enter(ksm);  in blk_ksm_get_slot_for_key()
216 		slot = blk_ksm_find_and_grab_keyslot(ksm, key);  in blk_ksm_get_slot_for_key()
218 			blk_ksm_hw_exit(ksm);  in blk_ksm_get_slot_for_key()
226 		if (!list_empty(&ksm->idle_slots))  in blk_ksm_get_slot_for_key()
229 		blk_ksm_hw_exit(ksm);  in blk_ksm_get_slot_for_key()
230 		wait_event(ksm->idle_slots_wait_queue,  in blk_ksm_get_slot_for_key()
231 			   !list_empty(&ksm->idle_slots));  in blk_ksm_get_slot_for_key()
234 	slot = list_first_entry(&ksm->idle_slots, struct blk_ksm_keyslot,  in blk_ksm_get_slot_for_key()
238 	err = ksm->ksm_ll_ops.keyslot_program(ksm, key, slot_idx);  in blk_ksm_get_slot_for_key()
240 		wake_up(&ksm->idle_slots_wait_queue);  in blk_ksm_get_slot_for_key()
241 		blk_ksm_hw_exit(ksm);  in blk_ksm_get_slot_for_key()
249 	hlist_add_head(&slot->hash_node, blk_ksm_hash_bucket_for_key(ksm, key));  in blk_ksm_get_slot_for_key()
255 	blk_ksm_hw_exit(ksm);  in blk_ksm_get_slot_for_key()
269 	struct blk_keyslot_manager *ksm;  in blk_ksm_put_slot()  local
275 	ksm = slot->ksm;  in blk_ksm_put_slot()
278 					&ksm->idle_slots_lock, flags)) {  in blk_ksm_put_slot()
279 		list_add_tail(&slot->idle_slot_node, &ksm->idle_slots);  in blk_ksm_put_slot()
280 		spin_unlock_irqrestore(&ksm->idle_slots_lock, flags);  in blk_ksm_put_slot()
281 		wake_up(&ksm->idle_slots_wait_queue);  in blk_ksm_put_slot()
295 bool blk_ksm_crypto_cfg_supported(struct blk_keyslot_manager *ksm,  in blk_ksm_crypto_cfg_supported()  argument
298 	if (!ksm)  in blk_ksm_crypto_cfg_supported()
300 	if (!(ksm->crypto_modes_supported[cfg->crypto_mode] &  in blk_ksm_crypto_cfg_supported()
303 	if (ksm->max_dun_bytes_supported < cfg->dun_bytes)  in blk_ksm_crypto_cfg_supported()
322 int blk_ksm_evict_key(struct blk_keyslot_manager *ksm,  in blk_ksm_evict_key()  argument
328 	blk_ksm_hw_enter(ksm);  in blk_ksm_evict_key()
329 	slot = blk_ksm_find_keyslot(ksm, key);  in blk_ksm_evict_key()
337 	err = ksm->ksm_ll_ops.keyslot_evict(ksm, key,  in blk_ksm_evict_key()
346 	blk_ksm_hw_exit(ksm);  in blk_ksm_evict_key()
359 void blk_ksm_reprogram_all_keys(struct blk_keyslot_manager *ksm)  in blk_ksm_reprogram_all_keys()  argument
364 	down_write(&ksm->lock);  in blk_ksm_reprogram_all_keys()
365 	for (slot = 0; slot < ksm->num_slots; slot++) {  in blk_ksm_reprogram_all_keys()
366 		const struct blk_crypto_key *key = ksm->slots[slot].key;  in blk_ksm_reprogram_all_keys()
372 		err = ksm->ksm_ll_ops.keyslot_program(ksm, key, slot);  in blk_ksm_reprogram_all_keys()
375 	up_write(&ksm->lock);  in blk_ksm_reprogram_all_keys()
379 void blk_ksm_destroy(struct blk_keyslot_manager *ksm)  in blk_ksm_destroy()  argument
381 	if (!ksm)  in blk_ksm_destroy()
383 	kvfree(ksm->slot_hashtable);  in blk_ksm_destroy()
384 	kvfree_sensitive(ksm->slots, sizeof(ksm->slots[0]) * ksm->num_slots);  in blk_ksm_destroy()
385 	memzero_explicit(ksm, sizeof(*ksm));  in blk_ksm_destroy()
389 bool blk_ksm_register(struct blk_keyslot_manager *ksm, struct request_queue *q)  in blk_ksm_register()  argument
395 	q->ksm = ksm;  in blk_ksm_register()
402 	q->ksm = NULL;  in blk_ksm_unregister()