166 static inline void dump_entry_trace(struct dma_debug_entry *entry)  in dump_entry_trace()  argument
169 	if (entry) {  in dump_entry_trace()
171 		stack_trace_print(entry->stack_entries, entry->stack_len, 0);  in dump_entry_trace()
217 #define err_printk(dev, entry, format, arg...) do {			\  argument
224 			dump_entry_trace(entry);			\
236 static int hash_fn(struct dma_debug_entry *entry)  in hash_fn()  argument
242 	return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK;  in hash_fn()
248 static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry,  in get_hash_bucket()  argument
252 	int idx = hash_fn(entry);  in get_hash_bucket()
290  * Search a given entry in the hash bucket list
296 	struct dma_debug_entry *entry, *ret = NULL;  in __hash_bucket_find()  local
299 	list_for_each_entry(entry, &bucket->list, list) {  in __hash_bucket_find()
300 		if (!match(ref, entry))  in __hash_bucket_find()
309 		 * best-fit algorithm here which returns the entry from  in __hash_bucket_find()
315 		entry->size         == ref->size         ? ++match_lvl : 0;  in __hash_bucket_find()
316 		entry->type         == ref->type         ? ++match_lvl : 0;  in __hash_bucket_find()
317 		entry->direction    == ref->direction    ? ++match_lvl : 0;  in __hash_bucket_find()
318 		entry->sg_call_ents == ref->sg_call_ents ? ++match_lvl : 0;  in __hash_bucket_find()
322 			return entry;  in __hash_bucket_find()
325 			 * We found an entry that fits better then the  in __hash_bucket_find()
329 			ret      = entry;  in __hash_bucket_find()
353 	struct dma_debug_entry *entry, index = *ref;  in bucket_find_contain()  local
357 		entry = __hash_bucket_find(*bucket, ref, containing_match);  in bucket_find_contain()
359 		if (entry)  in bucket_find_contain()
360 			return entry;  in bucket_find_contain()
374  * Add an entry to a hash bucket
377 			    struct dma_debug_entry *entry)  in hash_bucket_add()  argument
379 	list_add_tail(&entry->list, &bucket->list);  in hash_bucket_add()
383  * Remove entry from a hash bucket list
385 static void hash_bucket_del(struct dma_debug_entry *entry)  in hash_bucket_del()  argument
387 	list_del(&entry->list);  in hash_bucket_del()
390 static unsigned long long phys_addr(struct dma_debug_entry *entry)  in phys_addr()  argument
392 	if (entry->type == dma_debug_resource)  in phys_addr()
393 		return __pfn_to_phys(entry->pfn) + entry->offset;  in phys_addr()
395 	return page_to_phys(pfn_to_page(entry->pfn)) + entry->offset;  in phys_addr()
407 		struct dma_debug_entry *entry;  in debug_dma_dump_mappings()  local
412 		list_for_each_entry(entry, &bucket->list, list) {  in debug_dma_dump_mappings()
413 			if (!dev || dev == entry->dev) {  in debug_dma_dump_mappings()
414 				dev_info(entry->dev,  in debug_dma_dump_mappings()
416 					 type2name[entry->type], idx,  in debug_dma_dump_mappings()
417 					 phys_addr(entry), entry->pfn,  in debug_dma_dump_mappings()
418 					 entry->dev_addr, entry->size,  in debug_dma_dump_mappings()
419 					 dir2name[entry->direction],  in debug_dma_dump_mappings()
420 					 maperr2str[entry->map_err_type]);  in debug_dma_dump_mappings()
434  * dma_unmap_{single|sg|page} or dma_free_coherent delete the entry.  If
435  * the entry already exists at insertion time add a tag as a reference
445  * dma_active_cacheline entry to track per event.  dma_map_sg(), on the
455 static phys_addr_t to_cacheline_number(struct dma_debug_entry *entry)  in to_cacheline_number()  argument
457 	return (entry->pfn << CACHELINE_PER_PAGE_SHIFT) +  in to_cacheline_number()
458 		(entry->offset >> L1_CACHE_SHIFT);  in to_cacheline_number()
508 static int active_cacheline_insert(struct dma_debug_entry *entry)  in active_cacheline_insert()  argument
510 	phys_addr_t cln = to_cacheline_number(entry);  in active_cacheline_insert()
518 	if (entry->direction == DMA_TO_DEVICE)  in active_cacheline_insert()
522 	rc = radix_tree_insert(&dma_active_cacheline, cln, entry);  in active_cacheline_insert()
530 static void active_cacheline_remove(struct dma_debug_entry *entry)  in active_cacheline_remove()  argument
532 	phys_addr_t cln = to_cacheline_number(entry);  in active_cacheline_remove()
536 	if (entry->direction == DMA_TO_DEVICE)  in active_cacheline_remove()
550  * Wrapper function for adding an entry to the hash.
553 static void add_dma_entry(struct dma_debug_entry *entry, unsigned long attrs)  in add_dma_entry()  argument
559 	bucket = get_hash_bucket(entry, &flags);  in add_dma_entry()
560 	hash_bucket_add(bucket, entry);  in add_dma_entry()
563 	rc = active_cacheline_insert(entry);  in add_dma_entry()
568 		err_printk(entry->dev, entry,  in add_dma_entry()
575 	struct dma_debug_entry *entry;  in dma_debug_create_entries()  local
578 	entry = (void *)get_zeroed_page(gfp);  in dma_debug_create_entries()
579 	if (!entry)  in dma_debug_create_entries()
583 		list_add_tail(&entry[i].list, &free_entries);  in dma_debug_create_entries()
593 	struct dma_debug_entry *entry;  in __dma_entry_alloc()  local
595 	entry = list_entry(free_entries.next, struct dma_debug_entry, list);  in __dma_entry_alloc()
596 	list_del(&entry->list);  in __dma_entry_alloc()
597 	memset(entry, 0, sizeof(*entry));  in __dma_entry_alloc()
603 	return entry;  in __dma_entry_alloc()
625 	struct dma_debug_entry *entry;  in dma_entry_alloc()  local
639 	entry = __dma_entry_alloc();  in dma_entry_alloc()
644 	entry->stack_len = stack_trace_save(entry->stack_entries,  in dma_entry_alloc()
645 					    ARRAY_SIZE(entry->stack_entries),  in dma_entry_alloc()
648 	return entry;  in dma_entry_alloc()
651 static void dma_entry_free(struct dma_debug_entry *entry)  in dma_entry_free()  argument
655 	active_cacheline_remove(entry);  in dma_entry_free()
662 	list_add(&entry->list, &free_entries);  in dma_entry_free()
773 		struct dma_debug_entry *entry;  in dump_show()  local
777 		list_for_each_entry(entry, &bucket->list, list) {  in dump_show()
780 				   dev_name(entry->dev),  in dump_show()
781 				   dev_driver_string(entry->dev),  in dump_show()
782 				   type2name[entry->type], idx,  in dump_show()
783 				   phys_addr(entry), entry->pfn,  in dump_show()
784 				   entry->dev_addr, entry->size,  in dump_show()
785 				   dir2name[entry->direction],  in dump_show()
786 				   maperr2str[entry->map_err_type]);  in dump_show()
814 	struct dma_debug_entry *entry;  in device_dma_allocations()  local
820 		list_for_each_entry(entry, &dma_entry_hash[i].list, list) {  in device_dma_allocations()
821 			if (entry->dev == dev) {  in device_dma_allocations()
823 				*out_entry = entry;  in device_dma_allocations()
835 	struct dma_debug_entry *entry;  in dma_debug_device_change()  local
843 		count = device_dma_allocations(dev, &entry);  in dma_debug_device_change()
846 		err_printk(dev, entry, "device driver has pending "  in dma_debug_device_change()
852 			count, entry->dev_addr, entry->size,  in dma_debug_device_change()
853 			dir2name[entry->direction], type2name[entry->type]);  in dma_debug_device_change()
945 	struct dma_debug_entry *entry;  in check_unmap()  local
950 	entry = bucket_find_exact(bucket, ref);  in check_unmap()
952 	if (!entry) {  in check_unmap()
970 	if (ref->size != entry->size) {  in check_unmap()
971 		err_printk(ref->dev, entry, "device driver frees "  in check_unmap()
975 			   ref->dev_addr, entry->size, ref->size);  in check_unmap()
978 	if (ref->type != entry->type) {  in check_unmap()
979 		err_printk(ref->dev, entry, "device driver frees "  in check_unmap()
984 			   type2name[entry->type], type2name[ref->type]);  in check_unmap()
985 	} else if ((entry->type == dma_debug_coherent) &&  in check_unmap()
986 		   (phys_addr(ref) != phys_addr(entry))) {  in check_unmap()
987 		err_printk(ref->dev, entry, "device driver frees "  in check_unmap()
993 			   phys_addr(entry),  in check_unmap()
998 	    ref->sg_call_ents != entry->sg_call_ents) {  in check_unmap()
999 		err_printk(ref->dev, entry, "device driver frees "  in check_unmap()
1000 			   "DMA sg list with different entry count "  in check_unmap()
1002 			   entry->sg_call_ents, ref->sg_call_ents);  in check_unmap()
1009 	if (ref->direction != entry->direction) {  in check_unmap()
1010 		err_printk(ref->dev, entry, "device driver frees "  in check_unmap()
1015 			   dir2name[entry->direction],  in check_unmap()
1024 	if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {  in check_unmap()
1025 		err_printk(ref->dev, entry,  in check_unmap()
1030 			   type2name[entry->type]);  in check_unmap()
1033 	hash_bucket_del(entry);  in check_unmap()
1034 	dma_entry_free(entry);  in check_unmap()
1078 	struct dma_debug_entry *entry;  in check_sync()  local
1084 	entry = bucket_find_contain(&bucket, ref, &flags);  in check_sync()
1086 	if (!entry) {  in check_sync()
1094 	if (ref->size > entry->size) {  in check_sync()
1095 		err_printk(dev, entry, "device driver syncs"  in check_sync()
1100 				entry->dev_addr, entry->size,  in check_sync()
1104 	if (entry->direction == DMA_BIDIRECTIONAL)  in check_sync()
1107 	if (ref->direction != entry->direction) {  in check_sync()
1108 		err_printk(dev, entry, "device driver syncs "  in check_sync()
1112 				(unsigned long long)ref->dev_addr, entry->size,  in check_sync()
1113 				dir2name[entry->direction],  in check_sync()
1117 	if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) &&  in check_sync()
1119 		err_printk(dev, entry, "device driver syncs "  in check_sync()
1123 				(unsigned long long)ref->dev_addr, entry->size,  in check_sync()
1124 				dir2name[entry->direction],  in check_sync()
1127 	if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) &&  in check_sync()
1129 		err_printk(dev, entry, "device driver syncs "  in check_sync()
1133 				(unsigned long long)ref->dev_addr, entry->size,  in check_sync()
1134 				dir2name[entry->direction],  in check_sync()
1138 	    ref->sg_call_ents != entry->sg_call_ents) {  in check_sync()
1139 		err_printk(ref->dev, entry, "device driver syncs "  in check_sync()
1140 			   "DMA sg list with different entry count "  in check_sync()
1142 			   entry->sg_call_ents, ref->sg_call_ents);  in check_sync()
1195 	struct dma_debug_entry *entry;  in debug_dma_map_page()  local
1203 	entry = dma_entry_alloc();  in debug_dma_map_page()
1204 	if (!entry)  in debug_dma_map_page()
1207 	entry->dev       = dev;  in debug_dma_map_page()
1208 	entry->type      = dma_debug_single;  in debug_dma_map_page()
1209 	entry->pfn	 = page_to_pfn(page);  in debug_dma_map_page()
1210 	entry->offset	 = offset;  in debug_dma_map_page()
1211 	entry->dev_addr  = dma_addr;  in debug_dma_map_page()
1212 	entry->size      = size;  in debug_dma_map_page()
1213 	entry->direction = direction;  in debug_dma_map_page()
1214 	entry->map_err_type = MAP_ERR_NOT_CHECKED;  in debug_dma_map_page()
1224 	add_dma_entry(entry, attrs);  in debug_dma_map_page()
1230 	struct dma_debug_entry *entry;  in debug_dma_mapping_error()  local
1241 	list_for_each_entry(entry, &bucket->list, list) {  in debug_dma_mapping_error()
1242 		if (!exact_match(&ref, entry))  in debug_dma_mapping_error()
1251 		 * best-fit algorithm here which updates the first entry  in debug_dma_mapping_error()
1255 		if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {  in debug_dma_mapping_error()
1256 			entry->map_err_type = MAP_ERR_CHECKED;  in debug_dma_mapping_error()
1285 	struct dma_debug_entry *entry;  in debug_dma_map_sg()  local
1299 		entry = dma_entry_alloc();  in debug_dma_map_sg()
1300 		if (!entry)  in debug_dma_map_sg()
1303 		entry->type           = dma_debug_sg;  in debug_dma_map_sg()
1304 		entry->dev            = dev;  in debug_dma_map_sg()
1305 		entry->pfn	      = page_to_pfn(sg_page(s));  in debug_dma_map_sg()
1306 		entry->offset	      = s->offset;  in debug_dma_map_sg()
1307 		entry->size           = sg_dma_len(s);  in debug_dma_map_sg()
1308 		entry->dev_addr       = sg_dma_address(s);  in debug_dma_map_sg()
1309 		entry->direction      = direction;  in debug_dma_map_sg()
1310 		entry->sg_call_ents   = nents;  in debug_dma_map_sg()
1311 		entry->sg_mapped_ents = mapped_ents;  in debug_dma_map_sg()
1315 		add_dma_entry(entry, attrs);  in debug_dma_map_sg()
1322 	struct dma_debug_entry *entry;  in get_nr_mapped_entries()  local
1328 	entry        = bucket_find_exact(bucket, ref);  in get_nr_mapped_entries()
1331 	if (entry)  in get_nr_mapped_entries()
1332 		mapped_ents = entry->sg_mapped_ents;  in get_nr_mapped_entries()
1374 	struct dma_debug_entry *entry;  in debug_dma_alloc_coherent()  local
1386 	entry = dma_entry_alloc();  in debug_dma_alloc_coherent()
1387 	if (!entry)  in debug_dma_alloc_coherent()
1390 	entry->type      = dma_debug_coherent;  in debug_dma_alloc_coherent()
1391 	entry->dev       = dev;  in debug_dma_alloc_coherent()
1392 	entry->offset	 = offset_in_page(virt);  in debug_dma_alloc_coherent()
1393 	entry->size      = size;  in debug_dma_alloc_coherent()
1394 	entry->dev_addr  = dma_addr;  in debug_dma_alloc_coherent()
1395 	entry->direction = DMA_BIDIRECTIONAL;  in debug_dma_alloc_coherent()
1398 		entry->pfn = vmalloc_to_pfn(virt);  in debug_dma_alloc_coherent()
1400 		entry->pfn = page_to_pfn(virt_to_page(virt));  in debug_dma_alloc_coherent()
1402 	add_dma_entry(entry, attrs);  in debug_dma_alloc_coherent()
1436 	struct dma_debug_entry *entry;  in debug_dma_map_resource()  local
1441 	entry = dma_entry_alloc();  in debug_dma_map_resource()
1442 	if (!entry)  in debug_dma_map_resource()
1445 	entry->type		= dma_debug_resource;  in debug_dma_map_resource()
1446 	entry->dev		= dev;  in debug_dma_map_resource()
1447 	entry->pfn		= PHYS_PFN(addr);  in debug_dma_map_resource()
1448 	entry->offset		= offset_in_page(addr);  in debug_dma_map_resource()
1449 	entry->size		= size;  in debug_dma_map_resource()
1450 	entry->dev_addr		= dma_addr;  in debug_dma_map_resource()
1451 	entry->direction	= direction;  in debug_dma_map_resource()
1452 	entry->map_err_type	= MAP_ERR_NOT_CHECKED;  in debug_dma_map_resource()
1454 	add_dma_entry(entry, attrs);  in debug_dma_map_resource()