1 // SPDX-License-Identifier: GPL-2.0
7  * DOC: blk-crypto profiles
9  * 'struct blk_crypto_profile' contains all generic inline encryption-related
17  * these keyslots in a device-independent way, using the driver-provided
22  * For more information, see Documentation/block/inline-encryption.rst.
25 #define pr_fmt(fmt) "blk-crypto: " fmt
27 #include <linux/blk-crypto-profile.h>
34 #include <linux/blk-integrity.h>
47 	 * Calling into the driver requires profile->lock held and the device  in blk_crypto_hw_enter()
49 	 * and release profile->lock via blk_crypto_reprogram_all_keys().  in blk_crypto_hw_enter()
51 	if (profile->dev)  in blk_crypto_hw_enter()
52 		pm_runtime_get_sync(profile->dev);  in blk_crypto_hw_enter()
53 	down_write(&profile->lock);  in blk_crypto_hw_enter()
58 	up_write(&profile->lock);  in blk_crypto_hw_exit()
59 	if (profile->dev)  in blk_crypto_hw_exit()
60 		pm_runtime_put_sync(profile->dev);  in blk_crypto_hw_exit()
64  * blk_crypto_profile_init() - Initialize a blk_crypto_profile
81 	init_rwsem(&profile->lock);  in blk_crypto_profile_init()
88 	profile->slots = kvcalloc(num_slots, sizeof(profile->slots[0]),  in blk_crypto_profile_init()
90 	if (!profile->slots)  in blk_crypto_profile_init()
91 		return -ENOMEM;  in blk_crypto_profile_init()
93 	profile->num_slots = num_slots;  in blk_crypto_profile_init()
95 	init_waitqueue_head(&profile->idle_slots_wait_queue);  in blk_crypto_profile_init()
96 	INIT_LIST_HEAD(&profile->idle_slots);  in blk_crypto_profile_init()
99 		profile->slots[slot].profile = profile;  in blk_crypto_profile_init()
100 		list_add_tail(&profile->slots[slot].idle_slot_node,  in blk_crypto_profile_init()
101 			      &profile->idle_slots);  in blk_crypto_profile_init()
104 	spin_lock_init(&profile->idle_slots_lock);  in blk_crypto_profile_init()
108 	 * hash_ptr() assumes bits != 0, so ensure the hash table has at least 2  in blk_crypto_profile_init()
111 	if (slot_hashtable_size < 2)  in blk_crypto_profile_init()
112 		slot_hashtable_size = 2;  in blk_crypto_profile_init()
114 	profile->log_slot_ht_size = ilog2(slot_hashtable_size);  in blk_crypto_profile_init()
115 	profile->slot_hashtable =  in blk_crypto_profile_init()
117 			       sizeof(profile->slot_hashtable[0]), GFP_KERNEL);  in blk_crypto_profile_init()
118 	if (!profile->slot_hashtable)  in blk_crypto_profile_init()
121 		INIT_HLIST_HEAD(&profile->slot_hashtable[i]);  in blk_crypto_profile_init()
127 	return -ENOMEM;  in blk_crypto_profile_init()
137  * devm_blk_crypto_profile_init() - Resource-managed blk_crypto_profile_init()
166 	return &profile->slot_hashtable[  in blk_crypto_hash_bucket_for_key()
167 			hash_ptr(key, profile->log_slot_ht_size)];  in blk_crypto_hash_bucket_for_key()
173 	struct blk_crypto_profile *profile = slot->profile;  in blk_crypto_remove_slot_from_lru_list()
176 	spin_lock_irqsave(&profile->idle_slots_lock, flags);  in blk_crypto_remove_slot_from_lru_list()
177 	list_del(&slot->idle_slot_node);  in blk_crypto_remove_slot_from_lru_list()
178 	spin_unlock_irqrestore(&profile->idle_slots_lock, flags);  in blk_crypto_remove_slot_from_lru_list()
190 		if (slotp->key == key)  in blk_crypto_find_keyslot()
205 	if (atomic_inc_return(&slot->slot_refs) == 1) {  in blk_crypto_find_and_grab_keyslot()
213  * blk_crypto_keyslot_index() - Get the index of a keyslot
216  * Return: the 0-based index of the keyslot within the device's keyslots.
220 	return slot - slot->profile->slots;  in blk_crypto_keyslot_index()
225  * blk_crypto_get_keyslot() - Get a keyslot for a key, if needed.
237  * Context: Process context. Takes and releases profile->lock.
255 	if (profile->num_slots == 0)  in blk_crypto_get_keyslot()
258 	down_read(&profile->lock);  in blk_crypto_get_keyslot()
260 	up_read(&profile->lock);  in blk_crypto_get_keyslot()
276 		if (!list_empty(&profile->idle_slots))  in blk_crypto_get_keyslot()
280 		wait_event(profile->idle_slots_wait_queue,  in blk_crypto_get_keyslot()
281 			   !list_empty(&profile->idle_slots));  in blk_crypto_get_keyslot()
284 	slot = list_first_entry(&profile->idle_slots, struct blk_crypto_keyslot,  in blk_crypto_get_keyslot()
288 	err = profile->ll_ops.keyslot_program(profile, key, slot_idx);  in blk_crypto_get_keyslot()
290 		wake_up(&profile->idle_slots_wait_queue);  in blk_crypto_get_keyslot()
296 	if (slot->key)  in blk_crypto_get_keyslot()
297 		hlist_del(&slot->hash_node);  in blk_crypto_get_keyslot()
298 	slot->key = key;  in blk_crypto_get_keyslot()
299 	hlist_add_head(&slot->hash_node,  in blk_crypto_get_keyslot()
302 	atomic_set(&slot->slot_refs, 1);  in blk_crypto_get_keyslot()
313  * blk_crypto_put_keyslot() - Release a reference to a keyslot
326 	profile = slot->profile;  in blk_crypto_put_keyslot()
328 	if (atomic_dec_and_lock_irqsave(&slot->slot_refs,  in blk_crypto_put_keyslot()
329 					&profile->idle_slots_lock, flags)) {  in blk_crypto_put_keyslot()
330 		list_add_tail(&slot->idle_slot_node, &profile->idle_slots);  in blk_crypto_put_keyslot()
331 		spin_unlock_irqrestore(&profile->idle_slots_lock, flags);  in blk_crypto_put_keyslot()
332 		wake_up(&profile->idle_slots_wait_queue);  in blk_crypto_put_keyslot()
337  * __blk_crypto_cfg_supported() - Check whether the given crypto profile
349 	if (!(profile->modes_supported[cfg->crypto_mode] & cfg->data_unit_size))  in __blk_crypto_cfg_supported()
351 	if (profile->max_dun_bytes_supported < cfg->dun_bytes)  in __blk_crypto_cfg_supported()
357  * __blk_crypto_evict_key() - Evict a key from a device.
366  * allows layered devices to evict the key from their underlying devices.
368  * Context: Process context. Takes and releases profile->lock.
369  * Return: 0 on success or if there's no keyslot with the specified key, -EBUSY
370  *	   if the keyslot is still in use, or another -errno value on other
379 	if (profile->num_slots == 0) {  in __blk_crypto_evict_key()
380 		if (profile->ll_ops.keyslot_evict) {  in __blk_crypto_evict_key()
382 			err = profile->ll_ops.keyslot_evict(profile, key, -1);  in __blk_crypto_evict_key()
394 	if (WARN_ON_ONCE(atomic_read(&slot->slot_refs) != 0)) {  in __blk_crypto_evict_key()
395 		err = -EBUSY;  in __blk_crypto_evict_key()
398 	err = profile->ll_ops.keyslot_evict(profile, key,  in __blk_crypto_evict_key()
403 	hlist_del(&slot->hash_node);  in __blk_crypto_evict_key()
404 	slot->key = NULL;  in __blk_crypto_evict_key()
412  * blk_crypto_reprogram_all_keys() - Re-program all keyslots.
415  * Re-program all keyslots that are supposed to have a key programmed.  This is
418  * Context: Process context. Takes and releases profile->lock.
424 	if (profile->num_slots == 0)  in blk_crypto_reprogram_all_keys()
428 	down_write(&profile->lock);  in blk_crypto_reprogram_all_keys()
429 	for (slot = 0; slot < profile->num_slots; slot++) {  in blk_crypto_reprogram_all_keys()
430 		const struct blk_crypto_key *key = profile->slots[slot].key;  in blk_crypto_reprogram_all_keys()
436 		err = profile->ll_ops.keyslot_program(profile, key, slot);  in blk_crypto_reprogram_all_keys()
439 	up_write(&profile->lock);  in blk_crypto_reprogram_all_keys()
447 	kvfree(profile->slot_hashtable);  in blk_crypto_profile_destroy()
448 	kvfree_sensitive(profile->slots,  in blk_crypto_profile_destroy()
449 			 sizeof(profile->slots[0]) * profile->num_slots);  in blk_crypto_profile_destroy()
461 	q->crypto_profile = profile;  in blk_crypto_register()
467  * blk_crypto_intersect_capabilities() - restrict supported crypto capabilities
475  * Only use this when setting up the crypto profile for a layered device, before
484 		parent->max_dun_bytes_supported =  in blk_crypto_intersect_capabilities()
485 			min(parent->max_dun_bytes_supported,  in blk_crypto_intersect_capabilities()
486 			    child->max_dun_bytes_supported);  in blk_crypto_intersect_capabilities()
487 		for (i = 0; i < ARRAY_SIZE(child->modes_supported); i++)  in blk_crypto_intersect_capabilities()
488 			parent->modes_supported[i] &= child->modes_supported[i];  in blk_crypto_intersect_capabilities()
490 		parent->max_dun_bytes_supported = 0;  in blk_crypto_intersect_capabilities()
491 		memset(parent->modes_supported, 0,  in blk_crypto_intersect_capabilities()
492 		       sizeof(parent->modes_supported));  in blk_crypto_intersect_capabilities()
498  * blk_crypto_has_capabilities() - Check whether @target supports at least all
516 	for (i = 0; i < ARRAY_SIZE(target->modes_supported); i++) {  in blk_crypto_has_capabilities()
517 		if (reference->modes_supported[i] & ~target->modes_supported[i])  in blk_crypto_has_capabilities()
521 	if (reference->max_dun_bytes_supported >  in blk_crypto_has_capabilities()
522 	    target->max_dun_bytes_supported)  in blk_crypto_has_capabilities()
530  * blk_crypto_update_capabilities() - Update the capabilities of a crypto
537  * Blk-crypto requires that crypto capabilities that were
547  * for blk-crypto to see stale values - they only cause blk-crypto to
549  * might result in blk-crypto-fallback being used if available, or the bio being
555 	memcpy(dst->modes_supported, src->modes_supported,  in blk_crypto_update_capabilities()
556 	       sizeof(dst->modes_supported));  in blk_crypto_update_capabilities()
558 	dst->max_dun_bytes_supported = src->max_dun_bytes_supported;  in blk_crypto_update_capabilities()