166 static inline void dump_entry_trace(struct dma_debug_entry *entry)  in dump_entry_trace()  argument
169 	if (entry) {  in dump_entry_trace()
171 		stack_trace_print(entry->stack_entries, entry->stack_len, 0);  in dump_entry_trace()
217 #define err_printk(dev, entry, format, arg...) do {			\  argument
224 			dump_entry_trace(entry);			\
236 static int hash_fn(struct dma_debug_entry *entry)  in hash_fn()  argument
242 	return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK;  in hash_fn()
248 static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry,  in get_hash_bucket()  argument
252 	int idx = hash_fn(entry);  in get_hash_bucket()
290  * Search a given entry in the hash bucket list
296 	struct dma_debug_entry *entry, *ret = NULL;  in __hash_bucket_find()  local
299 	list_for_each_entry(entry, &bucket->list, list) {  in __hash_bucket_find()
300 		if (!match(ref, entry))  in __hash_bucket_find()
309 		 * best-fit algorithm here which returns the entry from  in __hash_bucket_find()
315 		entry->size         == ref->size         ? ++match_lvl : 0;  in __hash_bucket_find()
316 		entry->type         == ref->type         ? ++match_lvl : 0;  in __hash_bucket_find()
317 		entry->direction    == ref->direction    ? ++match_lvl : 0;  in __hash_bucket_find()
318 		entry->sg_call_ents == ref->sg_call_ents ? ++match_lvl : 0;  in __hash_bucket_find()
322 			return entry;  in __hash_bucket_find()
325 			 * We found an entry that fits better then the  in __hash_bucket_find()
329 			ret      = entry;  in __hash_bucket_find()
354 	struct dma_debug_entry *entry, index = *ref;  in bucket_find_contain()  local
358 		entry = __hash_bucket_find(*bucket, ref, containing_match);  in bucket_find_contain()
360 		if (entry)  in bucket_find_contain()
361 			return entry;  in bucket_find_contain()
376  * Add an entry to a hash bucket
379 			    struct dma_debug_entry *entry)  in hash_bucket_add()  argument
381 	list_add_tail(&entry->list, &bucket->list);  in hash_bucket_add()
385  * Remove entry from a hash bucket list
387 static void hash_bucket_del(struct dma_debug_entry *entry)  in hash_bucket_del()  argument
389 	list_del(&entry->list);  in hash_bucket_del()
392 static unsigned long long phys_addr(struct dma_debug_entry *entry)  in phys_addr()  argument
394 	if (entry->type == dma_debug_resource)  in phys_addr()
395 		return __pfn_to_phys(entry->pfn) + entry->offset;  in phys_addr()
397 	return page_to_phys(pfn_to_page(entry->pfn)) + entry->offset;  in phys_addr()
409 		struct dma_debug_entry *entry;  in debug_dma_dump_mappings()  local
414 		list_for_each_entry(entry, &bucket->list, list) {  in debug_dma_dump_mappings()
415 			if (!dev || dev == entry->dev) {  in debug_dma_dump_mappings()
416 				dev_info(entry->dev,  in debug_dma_dump_mappings()
418 					 type2name[entry->type], idx,  in debug_dma_dump_mappings()
419 					 phys_addr(entry), entry->pfn,  in debug_dma_dump_mappings()
420 					 entry->dev_addr, entry->size,  in debug_dma_dump_mappings()
421 					 dir2name[entry->direction],  in debug_dma_dump_mappings()
422 					 maperr2str[entry->map_err_type]);  in debug_dma_dump_mappings()
436  * dma_unmap_{single|sg|page} or dma_free_coherent delete the entry.  If
437  * the entry already exists at insertion time add a tag as a reference
447  * dma_active_cacheline entry to track per event.  dma_map_sg(), on the
457 static phys_addr_t to_cacheline_number(struct dma_debug_entry *entry)  in to_cacheline_number()  argument
459 	return (entry->pfn << CACHELINE_PER_PAGE_SHIFT) +  in to_cacheline_number()
460 		(entry->offset >> L1_CACHE_SHIFT);  in to_cacheline_number()
510 static int active_cacheline_insert(struct dma_debug_entry *entry)  in active_cacheline_insert()  argument
512 	phys_addr_t cln = to_cacheline_number(entry);  in active_cacheline_insert()
520 	if (entry->direction == DMA_TO_DEVICE)  in active_cacheline_insert()
524 	rc = radix_tree_insert(&dma_active_cacheline, cln, entry);  in active_cacheline_insert()
532 static void active_cacheline_remove(struct dma_debug_entry *entry)  in active_cacheline_remove()  argument
534 	phys_addr_t cln = to_cacheline_number(entry);  in active_cacheline_remove()
538 	if (entry->direction == DMA_TO_DEVICE)  in active_cacheline_remove()
552  * Wrapper function for adding an entry to the hash.
555 static void add_dma_entry(struct dma_debug_entry *entry, unsigned long attrs)  in add_dma_entry()  argument
561 	bucket = get_hash_bucket(entry, &flags);  in add_dma_entry()
562 	hash_bucket_add(bucket, entry);  in add_dma_entry()
565 	rc = active_cacheline_insert(entry);  in add_dma_entry()
570 		err_printk(entry->dev, entry,  in add_dma_entry()
577 	struct dma_debug_entry *entry;  in dma_debug_create_entries()  local
580 	entry = (void *)get_zeroed_page(gfp);  in dma_debug_create_entries()
581 	if (!entry)  in dma_debug_create_entries()
585 		list_add_tail(&entry[i].list, &free_entries);  in dma_debug_create_entries()
595 	struct dma_debug_entry *entry;  in __dma_entry_alloc()  local
597 	entry = list_entry(free_entries.next, struct dma_debug_entry, list);  in __dma_entry_alloc()
598 	list_del(&entry->list);  in __dma_entry_alloc()
599 	memset(entry, 0, sizeof(*entry));  in __dma_entry_alloc()
605 	return entry;  in __dma_entry_alloc()
627 	struct dma_debug_entry *entry;  in dma_entry_alloc()  local
641 	entry = __dma_entry_alloc();  in dma_entry_alloc()
646 	entry->stack_len = stack_trace_save(entry->stack_entries,  in dma_entry_alloc()
647 					    ARRAY_SIZE(entry->stack_entries),  in dma_entry_alloc()
650 	return entry;  in dma_entry_alloc()
653 static void dma_entry_free(struct dma_debug_entry *entry)  in dma_entry_free()  argument
657 	active_cacheline_remove(entry);  in dma_entry_free()
664 	list_add(&entry->list, &free_entries);  in dma_entry_free()
775 		struct dma_debug_entry *entry;  in dump_show()  local
779 		list_for_each_entry(entry, &bucket->list, list) {  in dump_show()
782 				   dev_name(entry->dev),  in dump_show()
783 				   dev_driver_string(entry->dev),  in dump_show()
784 				   type2name[entry->type], idx,  in dump_show()
785 				   phys_addr(entry), entry->pfn,  in dump_show()
786 				   entry->dev_addr, entry->size,  in dump_show()
787 				   dir2name[entry->direction],  in dump_show()
788 				   maperr2str[entry->map_err_type]);  in dump_show()
816 	struct dma_debug_entry *entry;  in device_dma_allocations()  local
822 		list_for_each_entry(entry, &dma_entry_hash[i].list, list) {  in device_dma_allocations()
823 			if (entry->dev == dev) {  in device_dma_allocations()
825 				*out_entry = entry;  in device_dma_allocations()
837 	struct dma_debug_entry *entry;  in dma_debug_device_change()  local
845 		count = device_dma_allocations(dev, &entry);  in dma_debug_device_change()
848 		err_printk(dev, entry, "device driver has pending "  in dma_debug_device_change()
854 			count, entry->dev_addr, entry->size,  in dma_debug_device_change()
855 			dir2name[entry->direction], type2name[entry->type]);  in dma_debug_device_change()
947 	struct dma_debug_entry *entry;  in check_unmap()  local
952 	entry = bucket_find_exact(bucket, ref);  in check_unmap()
954 	if (!entry) {  in check_unmap()
972 	if (ref->size != entry->size) {  in check_unmap()
973 		err_printk(ref->dev, entry, "device driver frees "  in check_unmap()
977 			   ref->dev_addr, entry->size, ref->size);  in check_unmap()
980 	if (ref->type != entry->type) {  in check_unmap()
981 		err_printk(ref->dev, entry, "device driver frees "  in check_unmap()
986 			   type2name[entry->type], type2name[ref->type]);  in check_unmap()
987 	} else if ((entry->type == dma_debug_coherent) &&  in check_unmap()
988 		   (phys_addr(ref) != phys_addr(entry))) {  in check_unmap()
989 		err_printk(ref->dev, entry, "device driver frees "  in check_unmap()
995 			   phys_addr(entry),  in check_unmap()
1000 	    ref->sg_call_ents != entry->sg_call_ents) {  in check_unmap()
1001 		err_printk(ref->dev, entry, "device driver frees "  in check_unmap()
1002 			   "DMA sg list with different entry count "  in check_unmap()
1004 			   entry->sg_call_ents, ref->sg_call_ents);  in check_unmap()
1011 	if (ref->direction != entry->direction) {  in check_unmap()
1012 		err_printk(ref->dev, entry, "device driver frees "  in check_unmap()
1017 			   dir2name[entry->direction],  in check_unmap()
1026 	if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {  in check_unmap()
1027 		err_printk(ref->dev, entry,  in check_unmap()
1032 			   type2name[entry->type]);  in check_unmap()
1035 	hash_bucket_del(entry);  in check_unmap()
1036 	dma_entry_free(entry);  in check_unmap()
1080 	struct dma_debug_entry *entry;  in check_sync()  local
1086 	entry = bucket_find_contain(&bucket, ref, &flags);  in check_sync()
1088 	if (!entry) {  in check_sync()
1096 	if (ref->size > entry->size) {  in check_sync()
1097 		err_printk(dev, entry, "device driver syncs"  in check_sync()
1102 				entry->dev_addr, entry->size,  in check_sync()
1106 	if (entry->direction == DMA_BIDIRECTIONAL)  in check_sync()
1109 	if (ref->direction != entry->direction) {  in check_sync()
1110 		err_printk(dev, entry, "device driver syncs "  in check_sync()
1114 				(unsigned long long)ref->dev_addr, entry->size,  in check_sync()
1115 				dir2name[entry->direction],  in check_sync()
1119 	if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) &&  in check_sync()
1121 		err_printk(dev, entry, "device driver syncs "  in check_sync()
1125 				(unsigned long long)ref->dev_addr, entry->size,  in check_sync()
1126 				dir2name[entry->direction],  in check_sync()
1129 	if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) &&  in check_sync()
1131 		err_printk(dev, entry, "device driver syncs "  in check_sync()
1135 				(unsigned long long)ref->dev_addr, entry->size,  in check_sync()
1136 				dir2name[entry->direction],  in check_sync()
1140 	    ref->sg_call_ents != entry->sg_call_ents) {  in check_sync()
1141 		err_printk(ref->dev, entry, "device driver syncs "  in check_sync()
1142 			   "DMA sg list with different entry count "  in check_sync()
1144 			   entry->sg_call_ents, ref->sg_call_ents);  in check_sync()
1197 	struct dma_debug_entry *entry;  in debug_dma_map_page()  local
1205 	entry = dma_entry_alloc();  in debug_dma_map_page()
1206 	if (!entry)  in debug_dma_map_page()
1209 	entry->dev       = dev;  in debug_dma_map_page()
1210 	entry->type      = dma_debug_single;  in debug_dma_map_page()
1211 	entry->pfn	 = page_to_pfn(page);  in debug_dma_map_page()
1212 	entry->offset	 = offset;  in debug_dma_map_page()
1213 	entry->dev_addr  = dma_addr;  in debug_dma_map_page()
1214 	entry->size      = size;  in debug_dma_map_page()
1215 	entry->direction = direction;  in debug_dma_map_page()
1216 	entry->map_err_type = MAP_ERR_NOT_CHECKED;  in debug_dma_map_page()
1226 	add_dma_entry(entry, attrs);  in debug_dma_map_page()
1232 	struct dma_debug_entry *entry;  in debug_dma_mapping_error()  local
1243 	list_for_each_entry(entry, &bucket->list, list) {  in debug_dma_mapping_error()
1244 		if (!exact_match(&ref, entry))  in debug_dma_mapping_error()
1253 		 * best-fit algorithm here which updates the first entry  in debug_dma_mapping_error()
1257 		if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {  in debug_dma_mapping_error()
1258 			entry->map_err_type = MAP_ERR_CHECKED;  in debug_dma_mapping_error()
1287 	struct dma_debug_entry *entry;  in debug_dma_map_sg()  local
1301 		entry = dma_entry_alloc();  in debug_dma_map_sg()
1302 		if (!entry)  in debug_dma_map_sg()
1305 		entry->type           = dma_debug_sg;  in debug_dma_map_sg()
1306 		entry->dev            = dev;  in debug_dma_map_sg()
1307 		entry->pfn	      = page_to_pfn(sg_page(s));  in debug_dma_map_sg()
1308 		entry->offset	      = s->offset;  in debug_dma_map_sg()
1309 		entry->size           = sg_dma_len(s);  in debug_dma_map_sg()
1310 		entry->dev_addr       = sg_dma_address(s);  in debug_dma_map_sg()
1311 		entry->direction      = direction;  in debug_dma_map_sg()
1312 		entry->sg_call_ents   = nents;  in debug_dma_map_sg()
1313 		entry->sg_mapped_ents = mapped_ents;  in debug_dma_map_sg()
1317 		add_dma_entry(entry, attrs);  in debug_dma_map_sg()
1324 	struct dma_debug_entry *entry;  in get_nr_mapped_entries()  local
1330 	entry        = bucket_find_exact(bucket, ref);  in get_nr_mapped_entries()
1333 	if (entry)  in get_nr_mapped_entries()
1334 		mapped_ents = entry->sg_mapped_ents;  in get_nr_mapped_entries()
1376 	struct dma_debug_entry *entry;  in debug_dma_alloc_coherent()  local
1388 	entry = dma_entry_alloc();  in debug_dma_alloc_coherent()
1389 	if (!entry)  in debug_dma_alloc_coherent()
1392 	entry->type      = dma_debug_coherent;  in debug_dma_alloc_coherent()
1393 	entry->dev       = dev;  in debug_dma_alloc_coherent()
1394 	entry->offset	 = offset_in_page(virt);  in debug_dma_alloc_coherent()
1395 	entry->size      = size;  in debug_dma_alloc_coherent()
1396 	entry->dev_addr  = dma_addr;  in debug_dma_alloc_coherent()
1397 	entry->direction = DMA_BIDIRECTIONAL;  in debug_dma_alloc_coherent()
1400 		entry->pfn = vmalloc_to_pfn(virt);  in debug_dma_alloc_coherent()
1402 		entry->pfn = page_to_pfn(virt_to_page(virt));  in debug_dma_alloc_coherent()
1404 	add_dma_entry(entry, attrs);  in debug_dma_alloc_coherent()
1438 	struct dma_debug_entry *entry;  in debug_dma_map_resource()  local
1443 	entry = dma_entry_alloc();  in debug_dma_map_resource()
1444 	if (!entry)  in debug_dma_map_resource()
1447 	entry->type		= dma_debug_resource;  in debug_dma_map_resource()
1448 	entry->dev		= dev;  in debug_dma_map_resource()
1449 	entry->pfn		= PHYS_PFN(addr);  in debug_dma_map_resource()
1450 	entry->offset		= offset_in_page(addr);  in debug_dma_map_resource()
1451 	entry->size		= size;  in debug_dma_map_resource()
1452 	entry->dev_addr		= dma_addr;  in debug_dma_map_resource()
1453 	entry->direction	= direction;  in debug_dma_map_resource()
1454 	entry->map_err_type	= MAP_ERR_NOT_CHECKED;  in debug_dma_map_resource()
1456 	add_dma_entry(entry, attrs);  in debug_dma_map_resource()