182 static inline void dump_entry_trace(struct dma_debug_entry *entry)  in dump_entry_trace()  argument
185 	if (entry) {  in dump_entry_trace()
187 		print_stack_trace(&entry->stacktrace, 0);  in dump_entry_trace()
233 #define err_printk(dev, entry, format, arg...) do {			\  argument
240 			dump_entry_trace(entry);			\
252 static int hash_fn(struct dma_debug_entry *entry)  in hash_fn()  argument
258 	return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK;  in hash_fn()
264 static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry,  in get_hash_bucket()  argument
268 	int idx = hash_fn(entry);  in get_hash_bucket()
314 	struct dma_debug_entry *entry, *ret = NULL;  in __hash_bucket_find()  local
317 	list_for_each_entry(entry, &bucket->list, list) {  in __hash_bucket_find()
318 		if (!match(ref, entry))  in __hash_bucket_find()
333 		entry->size         == ref->size         ? ++match_lvl : 0;  in __hash_bucket_find()
334 		entry->type         == ref->type         ? ++match_lvl : 0;  in __hash_bucket_find()
335 		entry->direction    == ref->direction    ? ++match_lvl : 0;  in __hash_bucket_find()
336 		entry->sg_call_ents == ref->sg_call_ents ? ++match_lvl : 0;  in __hash_bucket_find()
340 			return entry;  in __hash_bucket_find()
347 			ret      = entry;  in __hash_bucket_find()
372 	struct dma_debug_entry *entry, index = *ref;  in bucket_find_contain()  local
376 		entry = __hash_bucket_find(*bucket, ref, containing_match);  in bucket_find_contain()
378 		if (entry)  in bucket_find_contain()
379 			return entry;  in bucket_find_contain()
397 			    struct dma_debug_entry *entry)  in hash_bucket_add()  argument
399 	list_add_tail(&entry->list, &bucket->list);  in hash_bucket_add()
405 static void hash_bucket_del(struct dma_debug_entry *entry)  in hash_bucket_del()  argument
407 	list_del(&entry->list);  in hash_bucket_del()
410 static unsigned long long phys_addr(struct dma_debug_entry *entry)  in phys_addr()  argument
412 	if (entry->type == dma_debug_resource)  in phys_addr()
413 		return __pfn_to_phys(entry->pfn) + entry->offset;  in phys_addr()
415 	return page_to_phys(pfn_to_page(entry->pfn)) + entry->offset;  in phys_addr()
427 		struct dma_debug_entry *entry;  in debug_dma_dump_mappings()  local
432 		list_for_each_entry(entry, &bucket->list, list) {  in debug_dma_dump_mappings()
433 			if (!dev || dev == entry->dev) {  in debug_dma_dump_mappings()
434 				dev_info(entry->dev,  in debug_dma_dump_mappings()
436 					 type2name[entry->type], idx,  in debug_dma_dump_mappings()
437 					 phys_addr(entry), entry->pfn,  in debug_dma_dump_mappings()
438 					 entry->dev_addr, entry->size,  in debug_dma_dump_mappings()
439 					 dir2name[entry->direction],  in debug_dma_dump_mappings()
440 					 maperr2str[entry->map_err_type]);  in debug_dma_dump_mappings()
477 static phys_addr_t to_cacheline_number(struct dma_debug_entry *entry)  in to_cacheline_number()  argument
479 	return (entry->pfn << CACHELINE_PER_PAGE_SHIFT) +  in to_cacheline_number()
480 		(entry->offset >> L1_CACHE_SHIFT);  in to_cacheline_number()
533 static int active_cacheline_insert(struct dma_debug_entry *entry)  in active_cacheline_insert()  argument
535 	phys_addr_t cln = to_cacheline_number(entry);  in active_cacheline_insert()
543 	if (entry->direction == DMA_TO_DEVICE)  in active_cacheline_insert()
547 	rc = radix_tree_insert(&dma_active_cacheline, cln, entry);  in active_cacheline_insert()
555 static void active_cacheline_remove(struct dma_debug_entry *entry)  in active_cacheline_remove()  argument
557 	phys_addr_t cln = to_cacheline_number(entry);  in active_cacheline_remove()
561 	if (entry->direction == DMA_TO_DEVICE)  in active_cacheline_remove()
585 	struct dma_debug_entry *entry = NULL;  in debug_dma_assert_idle()  local
605 			entry = ents[i];  in debug_dma_assert_idle()
612 	if (!entry)  in debug_dma_assert_idle()
615 	cln = to_cacheline_number(entry);  in debug_dma_assert_idle()
616 	err_printk(entry->dev, entry,  in debug_dma_assert_idle()
625 static void add_dma_entry(struct dma_debug_entry *entry)  in add_dma_entry()  argument
631 	bucket = get_hash_bucket(entry, &flags);  in add_dma_entry()
632 	hash_bucket_add(bucket, entry);  in add_dma_entry()
635 	rc = active_cacheline_insert(entry);  in add_dma_entry()
648 	struct dma_debug_entry *entry;  in __dma_entry_alloc()  local
650 	entry = list_entry(free_entries.next, struct dma_debug_entry, list);  in __dma_entry_alloc()
651 	list_del(&entry->list);  in __dma_entry_alloc()
652 	memset(entry, 0, sizeof(*entry));  in __dma_entry_alloc()
658 	return entry;  in __dma_entry_alloc()
668 	struct dma_debug_entry *entry;  in dma_entry_alloc()  local
680 	entry = __dma_entry_alloc();  in dma_entry_alloc()
685 	entry->stacktrace.max_entries = DMA_DEBUG_STACKTRACE_ENTRIES;  in dma_entry_alloc()
686 	entry->stacktrace.entries = entry->st_entries;  in dma_entry_alloc()
687 	entry->stacktrace.skip = 2;  in dma_entry_alloc()
688 	save_stack_trace(&entry->stacktrace);  in dma_entry_alloc()
691 	return entry;  in dma_entry_alloc()
694 static void dma_entry_free(struct dma_debug_entry *entry)  in dma_entry_free()  argument
698 	active_cacheline_remove(entry);  in dma_entry_free()
705 	list_add(&entry->list, &free_entries);  in dma_entry_free()
714 	struct dma_debug_entry *entry;  in dma_debug_resize_entries()  local
725 			entry = kzalloc(sizeof(*entry), GFP_KERNEL);  in dma_debug_resize_entries()
726 			if (!entry)  in dma_debug_resize_entries()
729 			list_add_tail(&entry->list, &tmp);  in dma_debug_resize_entries()
741 			entry = __dma_entry_alloc();  in dma_debug_resize_entries()
742 			kfree(entry);  in dma_debug_resize_entries()
766 	struct dma_debug_entry *entry, *next_entry;  in prealloc_memory()  local
770 		entry = kzalloc(sizeof(*entry), GFP_KERNEL);  in prealloc_memory()
771 		if (!entry)  in prealloc_memory()
774 		list_add_tail(&entry->list, &free_entries);  in prealloc_memory()
786 	list_for_each_entry_safe(entry, next_entry, &free_entries, list) {  in prealloc_memory()
787 		list_del(&entry->list);  in prealloc_memory()
788 		kfree(entry);  in prealloc_memory()
944 	struct dma_debug_entry *entry;  in device_dma_allocations()  local
950 		list_for_each_entry(entry, &dma_entry_hash[i].list, list) {  in device_dma_allocations()
951 			if (entry->dev == dev) {  in device_dma_allocations()
953 				*out_entry = entry;  in device_dma_allocations()
965 	struct dma_debug_entry *uninitialized_var(entry);  in dma_debug_device_change()
973 		count = device_dma_allocations(dev, &entry);  in dma_debug_device_change()
976 		err_printk(dev, entry, "DMA-API: device driver has pending "  in dma_debug_device_change()
982 			count, entry->dev_addr, entry->size,  in dma_debug_device_change()
983 			dir2name[entry->direction], type2name[entry->type]);  in dma_debug_device_change()
1075 	struct dma_debug_entry *entry;  in check_unmap()  local
1080 	entry = bucket_find_exact(bucket, ref);  in check_unmap()
1082 	if (!entry) {  in check_unmap()
1100 	if (ref->size != entry->size) {  in check_unmap()
1101 		err_printk(ref->dev, entry, "DMA-API: device driver frees "  in check_unmap()
1105 			   ref->dev_addr, entry->size, ref->size);  in check_unmap()
1108 	if (ref->type != entry->type) {  in check_unmap()
1109 		err_printk(ref->dev, entry, "DMA-API: device driver frees "  in check_unmap()
1114 			   type2name[entry->type], type2name[ref->type]);  in check_unmap()
1115 	} else if ((entry->type == dma_debug_coherent) &&  in check_unmap()
1116 		   (phys_addr(ref) != phys_addr(entry))) {  in check_unmap()
1117 		err_printk(ref->dev, entry, "DMA-API: device driver frees "  in check_unmap()
1123 			   phys_addr(entry),  in check_unmap()
1128 	    ref->sg_call_ents != entry->sg_call_ents) {  in check_unmap()
1129 		err_printk(ref->dev, entry, "DMA-API: device driver frees "  in check_unmap()
1132 			   entry->sg_call_ents, ref->sg_call_ents);  in check_unmap()
1139 	if (ref->direction != entry->direction) {  in check_unmap()
1140 		err_printk(ref->dev, entry, "DMA-API: device driver frees "  in check_unmap()
1145 			   dir2name[entry->direction],  in check_unmap()
1154 	if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {  in check_unmap()
1155 		err_printk(ref->dev, entry,  in check_unmap()
1160 			   type2name[entry->type]);  in check_unmap()
1163 	hash_bucket_del(entry);  in check_unmap()
1164 	dma_entry_free(entry);  in check_unmap()
1218 	struct dma_debug_entry *entry;  in check_sync()  local
1224 	entry = bucket_find_contain(&bucket, ref, &flags);  in check_sync()
1226 	if (!entry) {  in check_sync()
1234 	if (ref->size > entry->size) {  in check_sync()
1235 		err_printk(dev, entry, "DMA-API: device driver syncs"  in check_sync()
1240 				entry->dev_addr, entry->size,  in check_sync()
1244 	if (entry->direction == DMA_BIDIRECTIONAL)  in check_sync()
1247 	if (ref->direction != entry->direction) {  in check_sync()
1248 		err_printk(dev, entry, "DMA-API: device driver syncs "  in check_sync()
1252 				(unsigned long long)ref->dev_addr, entry->size,  in check_sync()
1253 				dir2name[entry->direction],  in check_sync()
1257 	if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) &&  in check_sync()
1259 		err_printk(dev, entry, "DMA-API: device driver syncs "  in check_sync()
1263 				(unsigned long long)ref->dev_addr, entry->size,  in check_sync()
1264 				dir2name[entry->direction],  in check_sync()
1267 	if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) &&  in check_sync()
1269 		err_printk(dev, entry, "DMA-API: device driver syncs "  in check_sync()
1273 				(unsigned long long)ref->dev_addr, entry->size,  in check_sync()
1274 				dir2name[entry->direction],  in check_sync()
1278 	    ref->sg_call_ents != entry->sg_call_ents) {  in check_sync()
1279 		err_printk(ref->dev, entry, "DMA-API: device driver syncs "  in check_sync()
1282 			   entry->sg_call_ents, ref->sg_call_ents);  in check_sync()
1319 	struct dma_debug_entry *entry;  in debug_dma_map_page()  local
1327 	entry = dma_entry_alloc();  in debug_dma_map_page()
1328 	if (!entry)  in debug_dma_map_page()
1331 	entry->dev       = dev;  in debug_dma_map_page()
1332 	entry->type      = dma_debug_page;  in debug_dma_map_page()
1333 	entry->pfn	 = page_to_pfn(page);  in debug_dma_map_page()
1334 	entry->offset	 = offset,  in debug_dma_map_page()
1335 	entry->dev_addr  = dma_addr;  in debug_dma_map_page()
1336 	entry->size      = size;  in debug_dma_map_page()
1337 	entry->direction = direction;  in debug_dma_map_page()
1338 	entry->map_err_type = MAP_ERR_NOT_CHECKED;  in debug_dma_map_page()
1341 		entry->type = dma_debug_single;  in debug_dma_map_page()
1351 	add_dma_entry(entry);  in debug_dma_map_page()
1358 	struct dma_debug_entry *entry;  in debug_dma_mapping_error()  local
1369 	list_for_each_entry(entry, &bucket->list, list) {  in debug_dma_mapping_error()
1370 		if (!exact_match(&ref, entry))  in debug_dma_mapping_error()
1383 		if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {  in debug_dma_mapping_error()
1384 			entry->map_err_type = MAP_ERR_CHECKED;  in debug_dma_mapping_error()
1417 	struct dma_debug_entry *entry;  in debug_dma_map_sg()  local
1425 		entry = dma_entry_alloc();  in debug_dma_map_sg()
1426 		if (!entry)  in debug_dma_map_sg()
1429 		entry->type           = dma_debug_sg;  in debug_dma_map_sg()
1430 		entry->dev            = dev;  in debug_dma_map_sg()
1431 		entry->pfn	      = page_to_pfn(sg_page(s));  in debug_dma_map_sg()
1432 		entry->offset	      = s->offset,  in debug_dma_map_sg()
1433 		entry->size           = sg_dma_len(s);  in debug_dma_map_sg()
1434 		entry->dev_addr       = sg_dma_address(s);  in debug_dma_map_sg()
1435 		entry->direction      = direction;  in debug_dma_map_sg()
1436 		entry->sg_call_ents   = nents;  in debug_dma_map_sg()
1437 		entry->sg_mapped_ents = mapped_ents;  in debug_dma_map_sg()
1447 		add_dma_entry(entry);  in debug_dma_map_sg()
1455 	struct dma_debug_entry *entry;  in get_nr_mapped_entries()  local
1461 	entry        = bucket_find_exact(bucket, ref);  in get_nr_mapped_entries()
1464 	if (entry)  in get_nr_mapped_entries()
1465 		mapped_ents = entry->sg_mapped_ents;  in get_nr_mapped_entries()
1507 	struct dma_debug_entry *entry;  in debug_dma_alloc_coherent()  local
1519 	entry = dma_entry_alloc();  in debug_dma_alloc_coherent()
1520 	if (!entry)  in debug_dma_alloc_coherent()
1523 	entry->type      = dma_debug_coherent;  in debug_dma_alloc_coherent()
1524 	entry->dev       = dev;  in debug_dma_alloc_coherent()
1525 	entry->offset	 = offset_in_page(virt);  in debug_dma_alloc_coherent()
1526 	entry->size      = size;  in debug_dma_alloc_coherent()
1527 	entry->dev_addr  = dma_addr;  in debug_dma_alloc_coherent()
1528 	entry->direction = DMA_BIDIRECTIONAL;  in debug_dma_alloc_coherent()
1531 		entry->pfn = vmalloc_to_pfn(virt);  in debug_dma_alloc_coherent()
1533 		entry->pfn = page_to_pfn(virt_to_page(virt));  in debug_dma_alloc_coherent()
1535 	add_dma_entry(entry);  in debug_dma_alloc_coherent()
1570 	struct dma_debug_entry *entry;  in debug_dma_map_resource()  local
1575 	entry = dma_entry_alloc();  in debug_dma_map_resource()
1576 	if (!entry)  in debug_dma_map_resource()
1579 	entry->type		= dma_debug_resource;  in debug_dma_map_resource()
1580 	entry->dev		= dev;  in debug_dma_map_resource()
1581 	entry->pfn		= PHYS_PFN(addr);  in debug_dma_map_resource()
1582 	entry->offset		= offset_in_page(addr);  in debug_dma_map_resource()
1583 	entry->size		= size;  in debug_dma_map_resource()
1584 	entry->dev_addr		= dma_addr;  in debug_dma_map_resource()
1585 	entry->direction	= direction;  in debug_dma_map_resource()
1586 	entry->map_err_type	= MAP_ERR_NOT_CHECKED;  in debug_dma_map_resource()
1588 	add_dma_entry(entry);  in debug_dma_map_resource()