120 static void block_list_remove(struct sst_dsp *dsp,  in block_list_remove()  argument
132 				dev_err(dsp->dev,  in block_list_remove()
141 		list_move(&block->list, &dsp->free_block_list);  in block_list_remove()
142 		dev_dbg(dsp->dev, "block freed %d:%d at offset 0x%x\n",  in block_list_remove()
148 static int block_list_prepare(struct sst_dsp *dsp,  in block_list_prepare()  argument
160 				dev_err(dsp->dev,  in block_list_prepare()
212 	struct sst_dsp *dsp = (struct sst_dsp *)param;  in dma_chan_filter()  local
214 	return chan->device->dev == dsp->dma_dev;  in dma_chan_filter()
217 int sst_dsp_dma_get_channel(struct sst_dsp *dsp, int chan_id)  in sst_dsp_dma_get_channel()  argument
219 	struct sst_dma *dma = dsp->dma;  in sst_dsp_dma_get_channel()
228 	dma->ch = dma_request_channel(mask, dma_chan_filter, dsp);  in sst_dsp_dma_get_channel()
230 		dev_err(dsp->dev, "error: DMA request channel failed\n");  in sst_dsp_dma_get_channel()
242 		dev_err(dsp->dev, "error: unable to set DMA slave config %d\n",  in sst_dsp_dma_get_channel()
252 void sst_dsp_dma_put_channel(struct sst_dsp *dsp)  in sst_dsp_dma_put_channel()  argument
254 	struct sst_dma *dma = dsp->dma;  in sst_dsp_dma_put_channel()
330 struct sst_fw *sst_fw_new(struct sst_dsp *dsp,   in sst_fw_new()  argument
336 	if (!dsp->ops->parse_fw)  in sst_fw_new()
343 	sst_fw->dsp = dsp;  in sst_fw_new()
348 	sst_fw->dma_buf = dma_alloc_coherent(dsp->dma_dev, sst_fw->size,  in sst_fw_new()
351 		dev_err(dsp->dev, "error: DMA alloc failed\n");  in sst_fw_new()
359 	if (dsp->fw_use_dma) {  in sst_fw_new()
360 		err = sst_dsp_dma_get_channel(dsp, 0);  in sst_fw_new()
366 	err = dsp->ops->parse_fw(sst_fw);  in sst_fw_new()
368 		dev_err(dsp->dev, "error: parse fw failed %d\n", err);  in sst_fw_new()
372 	if (dsp->fw_use_dma)  in sst_fw_new()
373 		sst_dsp_dma_put_channel(dsp);  in sst_fw_new()
375 	mutex_lock(&dsp->mutex);  in sst_fw_new()
376 	list_add(&sst_fw->list, &dsp->fw_list);  in sst_fw_new()
377 	mutex_unlock(&dsp->mutex);  in sst_fw_new()
382 	if (dsp->fw_use_dma)  in sst_fw_new()
383 		sst_dsp_dma_put_channel(dsp);  in sst_fw_new()
385 	dma_free_coherent(dsp->dma_dev, sst_fw->size,  in sst_fw_new()
396 	struct sst_dsp *dsp = sst_fw->dsp;  in sst_fw_reload()  local
399 	dev_dbg(dsp->dev, "reloading firmware\n");  in sst_fw_reload()
402 	ret = dsp->ops->parse_fw(sst_fw);  in sst_fw_reload()
404 		dev_err(dsp->dev, "error: parse fw failed %d\n", ret);  in sst_fw_reload()
412 	struct sst_dsp *dsp = sst_fw->dsp;  in sst_fw_unload()  local
416 	dev_dbg(dsp->dev, "unloading firmware\n");  in sst_fw_unload()
418 	mutex_lock(&dsp->mutex);  in sst_fw_unload()
421 	list_for_each_entry_safe(module, mtmp, &dsp->module_list, list) {  in sst_fw_unload()
427 				block_list_remove(dsp, &runtime->block_list);  in sst_fw_unload()
433 			block_list_remove(dsp, &module->block_list);  in sst_fw_unload()
440 	block_list_remove(dsp, &dsp->scratch_block_list);  in sst_fw_unload()
442 	mutex_unlock(&dsp->mutex);  in sst_fw_unload()
449 	struct sst_dsp *dsp = sst_fw->dsp;  in sst_fw_free()  local
451 	mutex_lock(&dsp->mutex);  in sst_fw_free()
453 	mutex_unlock(&dsp->mutex);  in sst_fw_free()
456 		dma_free_coherent(dsp->dma_dev, sst_fw->size, sst_fw->dma_buf,  in sst_fw_free()
463 void sst_fw_free_all(struct sst_dsp *dsp)  in sst_fw_free_all()  argument
467 	mutex_lock(&dsp->mutex);  in sst_fw_free_all()
468 	list_for_each_entry_safe(sst_fw, t, &dsp->fw_list, list) {  in sst_fw_free_all()
471 		dma_free_coherent(dsp->dev, sst_fw->size, sst_fw->dma_buf,  in sst_fw_free_all()
475 	mutex_unlock(&dsp->mutex);  in sst_fw_free_all()
483 	struct sst_dsp *dsp = sst_fw->dsp;  in sst_module_new()  local
491 	sst_module->dsp = dsp;  in sst_module_new()
501 	mutex_lock(&dsp->mutex);  in sst_module_new()
502 	list_add(&sst_module->list, &dsp->module_list);  in sst_module_new()
503 	mutex_unlock(&dsp->mutex);  in sst_module_new()
512 	struct sst_dsp *dsp = sst_module->dsp;  in sst_module_free()  local
514 	mutex_lock(&dsp->mutex);  in sst_module_free()
516 	mutex_unlock(&dsp->mutex);  in sst_module_free()
525 	struct sst_dsp *dsp = module->dsp;  in sst_module_runtime_new()  local
533 	runtime->dsp = dsp;  in sst_module_runtime_new()
537 	mutex_lock(&dsp->mutex);  in sst_module_runtime_new()
539 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_new()
547 	struct sst_dsp *dsp = runtime->dsp;  in sst_module_runtime_free()  local
549 	mutex_lock(&dsp->mutex);  in sst_module_runtime_free()
551 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_free()
557 static struct sst_mem_block *find_block(struct sst_dsp *dsp,  in find_block()  argument
562 	list_for_each_entry(block, &dsp->free_block_list, list) {  in find_block()
571 static int block_alloc_contiguous(struct sst_dsp *dsp,  in block_alloc_contiguous()  argument
581 		block = find_block(dsp, ba);  in block_alloc_contiguous()
583 			list_splice(&tmp, &dsp->free_block_list);  in block_alloc_contiguous()
604 		dev_dbg(dsp->dev, "block allocated %d:%d at offset 0x%x\n",  in block_alloc_contiguous()
608 	list_splice(&tmp, &dsp->used_block_list);  in block_alloc_contiguous()
613 static int block_alloc(struct sst_dsp *dsp, struct sst_block_allocator *ba,  in block_alloc()  argument
623 	list_for_each_entry_safe(block, tmp, &dsp->free_block_list, list) {  in block_alloc()
635 		list_move(&block->list, &dsp->used_block_list);  in block_alloc()
636 		dev_dbg(dsp->dev, "block allocated %d:%d at offset 0x%x\n",  in block_alloc()
642 	list_for_each_entry_safe(block, tmp, &dsp->free_block_list, list) {  in block_alloc()
654 			ret = block_alloc_contiguous(dsp, ba, block_list);  in block_alloc()
665 int sst_alloc_blocks(struct sst_dsp *dsp, struct sst_block_allocator *ba,  in sst_alloc_blocks()  argument
670 	dev_dbg(dsp->dev, "block request 0x%x bytes at offset 0x%x type %d\n",  in sst_alloc_blocks()
673 	mutex_lock(&dsp->mutex);  in sst_alloc_blocks()
675 	ret = block_alloc(dsp, ba, block_list);  in sst_alloc_blocks()
677 		dev_err(dsp->dev, "error: can't alloc blocks %d\n", ret);  in sst_alloc_blocks()
682 	ret = block_list_prepare(dsp, block_list);  in sst_alloc_blocks()
684 		dev_err(dsp->dev, "error: prepare failed\n");  in sst_alloc_blocks()
687 	mutex_unlock(&dsp->mutex);  in sst_alloc_blocks()
692 int sst_free_blocks(struct sst_dsp *dsp, struct list_head *block_list)  in sst_free_blocks()  argument
694 	mutex_lock(&dsp->mutex);  in sst_free_blocks()
695 	block_list_remove(dsp, block_list);  in sst_free_blocks()
696 	mutex_unlock(&dsp->mutex);  in sst_free_blocks()
702 static int block_alloc_fixed(struct sst_dsp *dsp, struct sst_block_allocator *ba,  in block_alloc_fixed()  argument
733 			err = block_alloc_contiguous(dsp, &ba_tmp, block_list);  in block_alloc_fixed()
743 	list_for_each_entry_safe(block, tmp, &dsp->free_block_list, list) {  in block_alloc_fixed()
754 			list_move(&block->list, &dsp->used_block_list);  in block_alloc_fixed()
756 			dev_dbg(dsp->dev, "block allocated %d:%d at offset 0x%x\n",  in block_alloc_fixed()
765 			list_move(&block->list, &dsp->used_block_list);  in block_alloc_fixed()
771 			err = block_alloc_contiguous(dsp, &ba_tmp, block_list);  in block_alloc_fixed()
785 	struct sst_dsp *dsp = module->dsp;  in sst_module_alloc_blocks()  local
795 	dev_dbg(dsp->dev, "block request 0x%x bytes at offset 0x%x type %d\n",  in sst_module_alloc_blocks()
798 	mutex_lock(&dsp->mutex);  in sst_module_alloc_blocks()
801 	ret = block_alloc_fixed(dsp, &ba, &module->block_list);  in sst_module_alloc_blocks()
803 		dev_err(dsp->dev,  in sst_module_alloc_blocks()
806 		mutex_unlock(&dsp->mutex);  in sst_module_alloc_blocks()
811 	ret = block_list_prepare(dsp, &module->block_list);  in sst_module_alloc_blocks()
813 		dev_err(dsp->dev, "error: fw module prepare failed\n");  in sst_module_alloc_blocks()
818 	if (dsp->fw_use_dma) {  in sst_module_alloc_blocks()
819 		ret = sst_dsp_dma_copyto(dsp,  in sst_module_alloc_blocks()
820 			dsp->addr.lpe_base + module->offset,  in sst_module_alloc_blocks()
824 			dev_err(dsp->dev, "error: module copy failed\n");  in sst_module_alloc_blocks()
828 		sst_memcpy32(dsp->addr.lpe + module->offset, module->data,  in sst_module_alloc_blocks()
831 	mutex_unlock(&dsp->mutex);  in sst_module_alloc_blocks()
835 	block_list_remove(dsp, &module->block_list);  in sst_module_alloc_blocks()
836 	mutex_unlock(&dsp->mutex);  in sst_module_alloc_blocks()
844 	struct sst_dsp *dsp = module->dsp;  in sst_module_free_blocks()  local
846 	mutex_lock(&dsp->mutex);  in sst_module_free_blocks()
847 	block_list_remove(dsp, &module->block_list);  in sst_module_free_blocks()
848 	mutex_unlock(&dsp->mutex);  in sst_module_free_blocks()
856 	struct sst_dsp *dsp = runtime->dsp;  in sst_module_runtime_alloc_blocks()  local
868 	mutex_lock(&dsp->mutex);  in sst_module_runtime_alloc_blocks()
875 		dev_dbg(dsp->dev, "persistent fixed block request 0x%x bytes type %d offset 0x%x\n",  in sst_module_runtime_alloc_blocks()
879 		ret = block_alloc_fixed(dsp, &ba, &runtime->block_list);  in sst_module_runtime_alloc_blocks()
882 		dev_dbg(dsp->dev, "persistent block request 0x%x bytes type %d\n",  in sst_module_runtime_alloc_blocks()
886 		ret = block_alloc(dsp, &ba, &runtime->block_list);  in sst_module_runtime_alloc_blocks()
889 		dev_err(dsp->dev,  in sst_module_runtime_alloc_blocks()
892 		mutex_unlock(&dsp->mutex);  in sst_module_runtime_alloc_blocks()
898 	ret = block_list_prepare(dsp, &runtime->block_list);  in sst_module_runtime_alloc_blocks()
900 		dev_err(dsp->dev, "error: runtime block prepare failed\n");  in sst_module_runtime_alloc_blocks()
904 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_alloc_blocks()
908 	block_list_remove(dsp, &module->block_list);  in sst_module_runtime_alloc_blocks()
909 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_alloc_blocks()
916 	struct sst_dsp *dsp = runtime->dsp;  in sst_module_runtime_free_blocks()  local
918 	mutex_lock(&dsp->mutex);  in sst_module_runtime_free_blocks()
919 	block_list_remove(dsp, &runtime->block_list);  in sst_module_runtime_free_blocks()
920 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_free_blocks()
928 	struct sst_dsp *dsp = runtime->dsp;  in sst_module_runtime_save()  local
932 	dev_dbg(dsp->dev, "saving runtime %d memory at 0x%x size 0x%x\n",  in sst_module_runtime_save()
936 	context->buffer = dma_alloc_coherent(dsp->dma_dev,  in sst_module_runtime_save()
940 		dev_err(dsp->dev, "error: DMA context alloc failed\n");  in sst_module_runtime_save()
944 	mutex_lock(&dsp->mutex);  in sst_module_runtime_save()
946 	if (dsp->fw_use_dma) {  in sst_module_runtime_save()
948 		ret = sst_dsp_dma_get_channel(dsp, 0);  in sst_module_runtime_save()
952 		ret = sst_dsp_dma_copyfrom(dsp, context->dma_buffer,  in sst_module_runtime_save()
953 			dsp->addr.lpe_base + runtime->persistent_offset,  in sst_module_runtime_save()
955 		sst_dsp_dma_put_channel(dsp);  in sst_module_runtime_save()
957 			dev_err(dsp->dev, "error: context copy failed\n");  in sst_module_runtime_save()
961 		sst_memcpy32(context->buffer, dsp->addr.lpe +  in sst_module_runtime_save()
966 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_save()
974 	struct sst_dsp *dsp = runtime->dsp;  in sst_module_runtime_restore()  local
978 	dev_dbg(dsp->dev, "restoring runtime %d memory at 0x%x size 0x%x\n",  in sst_module_runtime_restore()
982 	mutex_lock(&dsp->mutex);  in sst_module_runtime_restore()
985 		dev_info(dsp->dev, "no context buffer need to restore!\n");  in sst_module_runtime_restore()
989 	if (dsp->fw_use_dma) {  in sst_module_runtime_restore()
991 		ret = sst_dsp_dma_get_channel(dsp, 0);  in sst_module_runtime_restore()
995 		ret = sst_dsp_dma_copyto(dsp,  in sst_module_runtime_restore()
996 			dsp->addr.lpe_base + runtime->persistent_offset,  in sst_module_runtime_restore()
998 		sst_dsp_dma_put_channel(dsp);  in sst_module_runtime_restore()
1000 			dev_err(dsp->dev, "error: module copy failed\n");  in sst_module_runtime_restore()
1004 		sst_memcpy32(dsp->addr.lpe + runtime->persistent_offset,  in sst_module_runtime_restore()
1007 	dma_free_coherent(dsp->dma_dev, module->persistent_size,  in sst_module_runtime_restore()
1012 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_restore()
1018 struct sst_mem_block *sst_mem_block_register(struct sst_dsp *dsp, u32 offset,  in sst_mem_block_register()  argument
1032 	block->dsp = dsp;  in sst_mem_block_register()
1036 	mutex_lock(&dsp->mutex);  in sst_mem_block_register()
1037 	list_add(&block->list, &dsp->free_block_list);  in sst_mem_block_register()
1038 	mutex_unlock(&dsp->mutex);  in sst_mem_block_register()
1045 void sst_mem_block_unregister_all(struct sst_dsp *dsp)  in sst_mem_block_unregister_all()  argument
1049 	mutex_lock(&dsp->mutex);  in sst_mem_block_unregister_all()
1052 	list_for_each_entry_safe(block, tmp, &dsp->used_block_list, list) {  in sst_mem_block_unregister_all()
1058 	list_for_each_entry_safe(block, tmp, &dsp->free_block_list, list) {  in sst_mem_block_unregister_all()
1063 	mutex_unlock(&dsp->mutex);  in sst_mem_block_unregister_all()
1068 int sst_block_alloc_scratch(struct sst_dsp *dsp)  in sst_block_alloc_scratch()  argument
1074 	mutex_lock(&dsp->mutex);  in sst_block_alloc_scratch()
1077 	dsp->scratch_size = 0;  in sst_block_alloc_scratch()
1078 	list_for_each_entry(module, &dsp->module_list, list) {  in sst_block_alloc_scratch()
1079 		dev_dbg(dsp->dev, "module %d scratch req 0x%x bytes\n",  in sst_block_alloc_scratch()
1081 		if (dsp->scratch_size < module->scratch_size)  in sst_block_alloc_scratch()
1082 			dsp->scratch_size = module->scratch_size;  in sst_block_alloc_scratch()
1085 	dev_dbg(dsp->dev, "scratch buffer required is 0x%x bytes\n",  in sst_block_alloc_scratch()
1086 		dsp->scratch_size);  in sst_block_alloc_scratch()
1088 	if (dsp->scratch_size == 0) {  in sst_block_alloc_scratch()
1089 		dev_info(dsp->dev, "no modules need scratch buffer\n");  in sst_block_alloc_scratch()
1090 		mutex_unlock(&dsp->mutex);  in sst_block_alloc_scratch()
1095 	dev_dbg(dsp->dev, "allocating scratch blocks\n");  in sst_block_alloc_scratch()
1097 	ba.size = dsp->scratch_size;  in sst_block_alloc_scratch()
1101 	if (dsp->scratch_offset != 0) {  in sst_block_alloc_scratch()
1103 		dev_dbg(dsp->dev, "block request 0x%x bytes type %d at 0x%x\n",  in sst_block_alloc_scratch()
1106 		ba.offset = dsp->scratch_offset;  in sst_block_alloc_scratch()
1109 		ret = block_alloc_fixed(dsp, &ba, &dsp->scratch_block_list);  in sst_block_alloc_scratch()
1112 		dev_dbg(dsp->dev, "block request 0x%x bytes type %d\n",  in sst_block_alloc_scratch()
1116 		ret = block_alloc(dsp, &ba, &dsp->scratch_block_list);  in sst_block_alloc_scratch()
1119 		dev_err(dsp->dev, "error: can't alloc scratch blocks\n");  in sst_block_alloc_scratch()
1120 		mutex_unlock(&dsp->mutex);  in sst_block_alloc_scratch()
1124 	ret = block_list_prepare(dsp, &dsp->scratch_block_list);  in sst_block_alloc_scratch()
1126 		dev_err(dsp->dev, "error: scratch block prepare failed\n");  in sst_block_alloc_scratch()
1127 		mutex_unlock(&dsp->mutex);  in sst_block_alloc_scratch()
1132 	dsp->scratch_offset = ba.offset;  in sst_block_alloc_scratch()
1133 	mutex_unlock(&dsp->mutex);  in sst_block_alloc_scratch()
1134 	return dsp->scratch_size;  in sst_block_alloc_scratch()
1139 void sst_block_free_scratch(struct sst_dsp *dsp)  in sst_block_free_scratch()  argument
1141 	mutex_lock(&dsp->mutex);  in sst_block_free_scratch()
1142 	block_list_remove(dsp, &dsp->scratch_block_list);  in sst_block_free_scratch()
1143 	mutex_unlock(&dsp->mutex);  in sst_block_free_scratch()
1148 struct sst_module *sst_module_get_from_id(struct sst_dsp *dsp, u32 id)  in sst_module_get_from_id()  argument
1152 	mutex_lock(&dsp->mutex);  in sst_module_get_from_id()
1154 	list_for_each_entry(module, &dsp->module_list, list) {  in sst_module_get_from_id()
1156 			mutex_unlock(&dsp->mutex);  in sst_module_get_from_id()
1161 	mutex_unlock(&dsp->mutex);  in sst_module_get_from_id()
1170 	struct sst_dsp *dsp = module->dsp;  in sst_module_runtime_get_from_id()  local
1172 	mutex_lock(&dsp->mutex);  in sst_module_runtime_get_from_id()
1176 			mutex_unlock(&dsp->mutex);  in sst_module_runtime_get_from_id()
1181 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_get_from_id()
1187 u32 sst_dsp_get_offset(struct sst_dsp *dsp, u32 offset,  in sst_dsp_get_offset()  argument
1192 		return offset - dsp->addr.iram_offset +  in sst_dsp_get_offset()
1193 			dsp->addr.dsp_iram_offset;  in sst_dsp_get_offset()
1195 		return offset - dsp->addr.dram_offset +  in sst_dsp_get_offset()
1196 			dsp->addr.dsp_dram_offset;  in sst_dsp_get_offset()