83 static inline struct fw_priv *to_fw_priv(struct kref *ref)  in to_fw_priv()
85 	return container_of(ref, struct fw_priv, ref);  in to_fw_priv()
97 void fw_state_init(struct fw_priv *fw_priv)  in fw_state_init()  argument
99 	struct fw_state *fw_st = &fw_priv->fw_st;  in fw_state_init()
105 static inline int fw_state_wait(struct fw_priv *fw_priv)  in fw_state_wait()  argument
107 	return __fw_state_wait_common(fw_priv, MAX_SCHEDULE_TIMEOUT);  in fw_state_wait()
110 static void fw_cache_piggyback_on_request(struct fw_priv *fw_priv);
112 static struct fw_priv *__allocate_fw_priv(const char *fw_name,  in __allocate_fw_priv()
119 	struct fw_priv *fw_priv;  in __allocate_fw_priv()  local
129 	fw_priv = kzalloc(sizeof(*fw_priv), GFP_ATOMIC);  in __allocate_fw_priv()
130 	if (!fw_priv)  in __allocate_fw_priv()
133 	fw_priv->fw_name = kstrdup_const(fw_name, GFP_ATOMIC);  in __allocate_fw_priv()
134 	if (!fw_priv->fw_name) {  in __allocate_fw_priv()
135 		kfree(fw_priv);  in __allocate_fw_priv()
139 	kref_init(&fw_priv->ref);  in __allocate_fw_priv()
140 	fw_priv->fwc = fwc;  in __allocate_fw_priv()
141 	fw_priv->data = dbuf;  in __allocate_fw_priv()
142 	fw_priv->allocated_size = size;  in __allocate_fw_priv()
143 	fw_priv->offset = offset;  in __allocate_fw_priv()
144 	fw_priv->opt_flags = opt_flags;  in __allocate_fw_priv()
145 	fw_state_init(fw_priv);  in __allocate_fw_priv()
147 	INIT_LIST_HEAD(&fw_priv->pending_list);  in __allocate_fw_priv()
150 	pr_debug("%s: fw-%s fw_priv=%p\n", __func__, fw_name, fw_priv);  in __allocate_fw_priv()
152 	return fw_priv;  in __allocate_fw_priv()
155 static struct fw_priv *__lookup_fw_priv(const char *fw_name)  in __lookup_fw_priv()
157 	struct fw_priv *tmp;  in __lookup_fw_priv()
168 			 struct fw_priv **fw_priv, void *dbuf, size_t size,  in alloc_lookup_fw_priv()  argument
171 	struct fw_priv *tmp;  in alloc_lookup_fw_priv()
183 			*fw_priv = tmp;  in alloc_lookup_fw_priv()
197 	*fw_priv = tmp;  in alloc_lookup_fw_priv()
205 	struct fw_priv *fw_priv = to_fw_priv(ref);  in __free_fw_priv()  local
206 	struct firmware_cache *fwc = fw_priv->fwc;  in __free_fw_priv()
209 		 __func__, fw_priv->fw_name, fw_priv, fw_priv->data,  in __free_fw_priv()
210 		 (unsigned int)fw_priv->size);  in __free_fw_priv()
212 	list_del(&fw_priv->list);  in __free_fw_priv()
215 	if (fw_is_paged_buf(fw_priv))  in __free_fw_priv()
216 		fw_free_paged_buf(fw_priv);  in __free_fw_priv()
217 	else if (!fw_priv->allocated_size)  in __free_fw_priv()
218 		vfree(fw_priv->data);  in __free_fw_priv()
220 	kfree_const(fw_priv->fw_name);  in __free_fw_priv()
221 	kfree(fw_priv);  in __free_fw_priv()
224 void free_fw_priv(struct fw_priv *fw_priv)  in free_fw_priv()  argument
226 	struct firmware_cache *fwc = fw_priv->fwc;  in free_fw_priv()
228 	if (!kref_put(&fw_priv->ref, __free_fw_priv))  in free_fw_priv()
233 bool fw_is_paged_buf(struct fw_priv *fw_priv)  in fw_is_paged_buf()  argument
235 	return fw_priv->is_paged_buf;  in fw_is_paged_buf()
238 void fw_free_paged_buf(struct fw_priv *fw_priv)  in fw_free_paged_buf()  argument
242 	if (!fw_priv->pages)  in fw_free_paged_buf()
245 	vunmap(fw_priv->data);  in fw_free_paged_buf()
247 	for (i = 0; i < fw_priv->nr_pages; i++)  in fw_free_paged_buf()
248 		__free_page(fw_priv->pages[i]);  in fw_free_paged_buf()
249 	kvfree(fw_priv->pages);  in fw_free_paged_buf()
250 	fw_priv->pages = NULL;  in fw_free_paged_buf()
251 	fw_priv->page_array_size = 0;  in fw_free_paged_buf()
252 	fw_priv->nr_pages = 0;  in fw_free_paged_buf()
253 	fw_priv->data = NULL;  in fw_free_paged_buf()
254 	fw_priv->size = 0;  in fw_free_paged_buf()
257 int fw_grow_paged_buf(struct fw_priv *fw_priv, int pages_needed)  in fw_grow_paged_buf()  argument
260 	if (fw_priv->page_array_size < pages_needed) {  in fw_grow_paged_buf()
262 					 fw_priv->page_array_size * 2);  in fw_grow_paged_buf()
269 		memcpy(new_pages, fw_priv->pages,  in fw_grow_paged_buf()
270 		       fw_priv->page_array_size * sizeof(void *));  in fw_grow_paged_buf()
271 		memset(&new_pages[fw_priv->page_array_size], 0, sizeof(void *) *  in fw_grow_paged_buf()
272 		       (new_array_size - fw_priv->page_array_size));  in fw_grow_paged_buf()
273 		kvfree(fw_priv->pages);  in fw_grow_paged_buf()
274 		fw_priv->pages = new_pages;  in fw_grow_paged_buf()
275 		fw_priv->page_array_size = new_array_size;  in fw_grow_paged_buf()
278 	while (fw_priv->nr_pages < pages_needed) {  in fw_grow_paged_buf()
279 		fw_priv->pages[fw_priv->nr_pages] =  in fw_grow_paged_buf()
282 		if (!fw_priv->pages[fw_priv->nr_pages])  in fw_grow_paged_buf()
284 		fw_priv->nr_pages++;  in fw_grow_paged_buf()
290 int fw_map_paged_buf(struct fw_priv *fw_priv)  in fw_map_paged_buf()  argument
293 	if (!fw_priv->pages)  in fw_map_paged_buf()
296 	vunmap(fw_priv->data);  in fw_map_paged_buf()
297 	fw_priv->data = vmap(fw_priv->pages, fw_priv->nr_pages, 0,  in fw_map_paged_buf()
299 	if (!fw_priv->data)  in fw_map_paged_buf()
310 static int fw_decompress_zstd(struct device *dev, struct fw_priv *fw_priv,  in fw_decompress_zstd()  argument
318 	if (fw_priv->allocated_size) {  in fw_decompress_zstd()
319 		out_size = fw_priv->allocated_size;  in fw_decompress_zstd()
320 		out_buf = fw_priv->data;  in fw_decompress_zstd()
357 	if (!fw_priv->allocated_size)  in fw_decompress_zstd()
358 		fw_priv->data = out_buf;  in fw_decompress_zstd()
359 	fw_priv->size = len;  in fw_decompress_zstd()
364 	if (err && !fw_priv->allocated_size)  in fw_decompress_zstd()
385 static int fw_decompress_xz_single(struct device *dev, struct fw_priv *fw_priv,  in fw_decompress_xz_single()  argument
399 	xz_buf.out_size = fw_priv->allocated_size;  in fw_decompress_xz_single()
400 	xz_buf.out = fw_priv->data;  in fw_decompress_xz_single()
406 	fw_priv->size = xz_buf.out_pos;  in fw_decompress_xz_single()
411 static int fw_decompress_xz_pages(struct device *dev, struct fw_priv *fw_priv,  in fw_decompress_xz_pages()  argument
428 	fw_priv->is_paged_buf = true;  in fw_decompress_xz_pages()
429 	fw_priv->size = 0;  in fw_decompress_xz_pages()
431 		if (fw_grow_paged_buf(fw_priv, fw_priv->nr_pages + 1)) {  in fw_decompress_xz_pages()
437 		page = fw_priv->pages[fw_priv->nr_pages - 1];  in fw_decompress_xz_pages()
443 		fw_priv->size += xz_buf.out_pos;  in fw_decompress_xz_pages()
451 		err = fw_map_paged_buf(fw_priv);  in fw_decompress_xz_pages()
458 static int fw_decompress_xz(struct device *dev, struct fw_priv *fw_priv,  in fw_decompress_xz()  argument
462 	if (fw_priv->data)  in fw_decompress_xz()
463 		return fw_decompress_xz_single(dev, fw_priv, in_size, in_buffer);  in fw_decompress_xz()
465 		return fw_decompress_xz_pages(dev, fw_priv, in_size, in_buffer);  in fw_decompress_xz()
488 fw_get_filesystem_firmware(struct device *device, struct fw_priv *fw_priv,  in fw_get_filesystem_firmware()  argument
491 					     struct fw_priv *fw_priv,  in fw_get_filesystem_firmware()
503 	if (!decompress && fw_priv->data) {  in fw_get_filesystem_firmware()
504 		buffer = fw_priv->data;  in fw_get_filesystem_firmware()
505 		msize = fw_priv->allocated_size;  in fw_get_filesystem_firmware()
522 			       fw_path[i], fw_priv->fw_name, suffix);  in fw_get_filesystem_firmware()
528 		fw_priv->size = 0;  in fw_get_filesystem_firmware()
535 		if ((fw_priv->opt_flags & FW_OPT_PARTIAL) && buffer)  in fw_get_filesystem_firmware()
539 		rc = kernel_read_file_from_path_initns(path, fw_priv->offset,  in fw_get_filesystem_firmware()
558 				fw_priv->fw_name);  in fw_get_filesystem_firmware()
559 			rc = decompress(device, fw_priv, size, buffer);  in fw_get_filesystem_firmware()
564 				fw_free_paged_buf(fw_priv);  in fw_get_filesystem_firmware()
569 				fw_priv->fw_name);  in fw_get_filesystem_firmware()
570 			if (!fw_priv->data)  in fw_get_filesystem_firmware()
571 				fw_priv->data = buffer;  in fw_get_filesystem_firmware()
572 			fw_priv->size = size;  in fw_get_filesystem_firmware()
574 		fw_state_done(fw_priv);  in fw_get_filesystem_firmware()
594 static void fw_set_page_data(struct fw_priv *fw_priv, struct firmware *fw)  in fw_set_page_data()  argument
596 	fw->priv = fw_priv;  in fw_set_page_data()
597 	fw->size = fw_priv->size;  in fw_set_page_data()
598 	fw->data = fw_priv->data;  in fw_set_page_data()
601 		 __func__, fw_priv->fw_name, fw_priv, fw_priv->data,  in fw_set_page_data()
602 		 (unsigned int)fw_priv->size);  in fw_set_page_data()
683 	struct fw_priv *fw_priv = fw->priv;  in assign_fw()  local
687 	if (!fw_priv->size || fw_state_is_aborted(fw_priv)) {  in assign_fw()
700 	if (device && (fw_priv->opt_flags & FW_OPT_UEVENT) &&  in assign_fw()
701 	    !(fw_priv->opt_flags & FW_OPT_NOCACHE)) {  in assign_fw()
702 		ret = fw_add_devm_name(device, fw_priv->fw_name);  in assign_fw()
713 	if (!(fw_priv->opt_flags & FW_OPT_NOCACHE) &&  in assign_fw()
714 	    fw_priv->fwc->state == FW_LOADER_START_CACHE)  in assign_fw()
715 		fw_cache_piggyback_on_request(fw_priv);  in assign_fw()
718 	fw_set_page_data(fw_priv, fw);  in assign_fw()
733 	struct fw_priv *fw_priv;  in _request_firmware_prepare()  local
748 	ret = alloc_lookup_fw_priv(name, &fw_cache, &fw_priv, dbuf, size,  in _request_firmware_prepare()
755 	firmware->priv = fw_priv;  in _request_firmware_prepare()
758 		ret = fw_state_wait(fw_priv);  in _request_firmware_prepare()
760 			fw_set_page_data(fw_priv, firmware);  in _request_firmware_prepare()
781 	struct fw_priv *fw_priv;  in fw_abort_batch_reqs()  local
787 	fw_priv = fw->priv;  in fw_abort_batch_reqs()
789 	if (!fw_state_is_aborted(fw_priv))  in fw_abort_batch_reqs()
790 		fw_state_aborted(fw_priv);  in fw_abort_batch_reqs()
1214 static struct fw_priv *lookup_fw_priv(const char *fw_name)  in lookup_fw_priv()
1216 	struct fw_priv *tmp;  in lookup_fw_priv()
1239 	struct fw_priv *fw_priv;  in uncache_firmware()  local
1247 	fw_priv = lookup_fw_priv(fw_name);  in uncache_firmware()
1248 	if (fw_priv) {  in uncache_firmware()
1249 		free_fw_priv(fw_priv);  in uncache_firmware()
1286 static void fw_cache_piggyback_on_request(struct fw_priv *fw_priv)  in fw_cache_piggyback_on_request()  argument
1288 	const char *name = fw_priv->fw_name;  in fw_cache_piggyback_on_request()
1289 	struct firmware_cache *fwc = fw_priv->fwc;  in fw_cache_piggyback_on_request()
1299 		kref_get(&fw_priv->ref);  in fw_cache_piggyback_on_request()
1534 static void fw_cache_piggyback_on_request(struct fw_priv *fw_priv)  in fw_cache_piggyback_on_request()  argument