Lines Matching full:entry
167 static inline void dump_entry_trace(struct dma_debug_entry *entry) in dump_entry_trace() argument
170 if (entry) { in dump_entry_trace()
172 stack_trace_print(entry->stack_entries, entry->stack_len, 0); in dump_entry_trace()
218 #define err_printk(dev, entry, format, arg...) do { \ argument
225 dump_entry_trace(entry); \
237 static int hash_fn(struct dma_debug_entry *entry) in hash_fn() argument
243 return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK; in hash_fn()
249 static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry, in get_hash_bucket() argument
253 int idx = hash_fn(entry); in get_hash_bucket()
291 * Search a given entry in the hash bucket list
297 struct dma_debug_entry *entry, *ret = NULL; in __hash_bucket_find() local
300 list_for_each_entry(entry, &bucket->list, list) { in __hash_bucket_find()
301 if (!match(ref, entry)) in __hash_bucket_find()
310 * best-fit algorithm here which returns the entry from in __hash_bucket_find()
316 entry->size == ref->size ? ++match_lvl : 0; in __hash_bucket_find()
317 entry->type == ref->type ? ++match_lvl : 0; in __hash_bucket_find()
318 entry->direction == ref->direction ? ++match_lvl : 0; in __hash_bucket_find()
319 entry->sg_call_ents == ref->sg_call_ents ? ++match_lvl : 0; in __hash_bucket_find()
323 return entry; in __hash_bucket_find()
326 * We found an entry that fits better then the in __hash_bucket_find()
330 ret = entry; in __hash_bucket_find()
354 struct dma_debug_entry *entry, index = *ref; in bucket_find_contain() local
358 entry = __hash_bucket_find(*bucket, ref, containing_match); in bucket_find_contain()
360 if (entry) in bucket_find_contain()
361 return entry; in bucket_find_contain()
375 * Add an entry to a hash bucket
378 struct dma_debug_entry *entry) in hash_bucket_add() argument
380 list_add_tail(&entry->list, &bucket->list); in hash_bucket_add()
384 * Remove entry from a hash bucket list
386 static void hash_bucket_del(struct dma_debug_entry *entry) in hash_bucket_del() argument
388 list_del(&entry->list); in hash_bucket_del()
391 static unsigned long long phys_addr(struct dma_debug_entry *entry) in phys_addr() argument
393 if (entry->type == dma_debug_resource) in phys_addr()
394 return __pfn_to_phys(entry->pfn) + entry->offset; in phys_addr()
396 return page_to_phys(pfn_to_page(entry->pfn)) + entry->offset; in phys_addr()
404 * dma_unmap_{single|sg|page} or dma_free_coherent delete the entry. If
405 * the entry already exists at insertion time add a tag as a reference
415 * dma_active_cacheline entry to track per event. dma_map_sg(), on the
425 static phys_addr_t to_cacheline_number(struct dma_debug_entry *entry) in to_cacheline_number() argument
427 return (entry->pfn << CACHELINE_PER_PAGE_SHIFT) + in to_cacheline_number()
428 (entry->offset >> L1_CACHE_SHIFT); in to_cacheline_number()
478 static int active_cacheline_insert(struct dma_debug_entry *entry) in active_cacheline_insert() argument
480 phys_addr_t cln = to_cacheline_number(entry); in active_cacheline_insert()
488 if (entry->direction == DMA_TO_DEVICE) in active_cacheline_insert()
492 rc = radix_tree_insert(&dma_active_cacheline, cln, entry); in active_cacheline_insert()
500 static void active_cacheline_remove(struct dma_debug_entry *entry) in active_cacheline_remove() argument
502 phys_addr_t cln = to_cacheline_number(entry); in active_cacheline_remove()
506 if (entry->direction == DMA_TO_DEVICE) in active_cacheline_remove()
529 struct dma_debug_entry *entry; in debug_dma_dump_mappings() local
533 list_for_each_entry(entry, &bucket->list, list) { in debug_dma_dump_mappings()
534 if (!dev || dev == entry->dev) { in debug_dma_dump_mappings()
535 cln = to_cacheline_number(entry); in debug_dma_dump_mappings()
536 dev_info(entry->dev, in debug_dma_dump_mappings()
538 type2name[entry->type], idx, in debug_dma_dump_mappings()
539 phys_addr(entry), entry->pfn, in debug_dma_dump_mappings()
540 entry->dev_addr, entry->size, in debug_dma_dump_mappings()
541 &cln, dir2name[entry->direction], in debug_dma_dump_mappings()
542 maperr2str[entry->map_err_type]); in debug_dma_dump_mappings()
561 struct dma_debug_entry *entry; in dump_show() local
565 list_for_each_entry(entry, &bucket->list, list) { in dump_show()
566 cln = to_cacheline_number(entry); in dump_show()
569 dev_driver_string(entry->dev), in dump_show()
570 dev_name(entry->dev), in dump_show()
571 type2name[entry->type], idx, in dump_show()
572 phys_addr(entry), entry->pfn, in dump_show()
573 entry->dev_addr, entry->size, in dump_show()
574 &cln, dir2name[entry->direction], in dump_show()
575 maperr2str[entry->map_err_type]); in dump_show()
584 * Wrapper function for adding an entry to the hash.
587 static void add_dma_entry(struct dma_debug_entry *entry, unsigned long attrs) in add_dma_entry() argument
593 bucket = get_hash_bucket(entry, &flags); in add_dma_entry()
594 hash_bucket_add(bucket, entry); in add_dma_entry()
597 rc = active_cacheline_insert(entry); in add_dma_entry()
602 err_printk(entry->dev, entry, in add_dma_entry()
609 struct dma_debug_entry *entry; in dma_debug_create_entries() local
612 entry = (void *)get_zeroed_page(gfp); in dma_debug_create_entries()
613 if (!entry) in dma_debug_create_entries()
617 list_add_tail(&entry[i].list, &free_entries); in dma_debug_create_entries()
627 struct dma_debug_entry *entry; in __dma_entry_alloc() local
629 entry = list_entry(free_entries.next, struct dma_debug_entry, list); in __dma_entry_alloc()
630 list_del(&entry->list); in __dma_entry_alloc()
631 memset(entry, 0, sizeof(*entry)); in __dma_entry_alloc()
637 return entry; in __dma_entry_alloc()
664 struct dma_debug_entry *entry; in dma_entry_alloc() local
680 entry = __dma_entry_alloc(); in dma_entry_alloc()
688 entry->stack_len = stack_trace_save(entry->stack_entries, in dma_entry_alloc()
689 ARRAY_SIZE(entry->stack_entries), in dma_entry_alloc()
692 return entry; in dma_entry_alloc()
695 static void dma_entry_free(struct dma_debug_entry *entry) in dma_entry_free() argument
699 active_cacheline_remove(entry); in dma_entry_free()
706 list_add(&entry->list, &free_entries); in dma_entry_free()
831 struct dma_debug_entry *entry; in device_dma_allocations() local
837 list_for_each_entry(entry, &dma_entry_hash[i].list, list) { in device_dma_allocations()
838 if (entry->dev == dev) { in device_dma_allocations()
840 *out_entry = entry; in device_dma_allocations()
852 struct dma_debug_entry *entry; in dma_debug_device_change() local
860 count = device_dma_allocations(dev, &entry); in dma_debug_device_change()
863 err_printk(dev, entry, "device driver has pending " in dma_debug_device_change()
869 count, entry->dev_addr, entry->size, in dma_debug_device_change()
870 dir2name[entry->direction], type2name[entry->type]); in dma_debug_device_change()
962 struct dma_debug_entry *entry; in check_unmap() local
967 entry = bucket_find_exact(bucket, ref); in check_unmap()
969 if (!entry) { in check_unmap()
987 if (ref->size != entry->size) { in check_unmap()
988 err_printk(ref->dev, entry, "device driver frees " in check_unmap()
992 ref->dev_addr, entry->size, ref->size); in check_unmap()
995 if (ref->type != entry->type) { in check_unmap()
996 err_printk(ref->dev, entry, "device driver frees " in check_unmap()
1001 type2name[entry->type], type2name[ref->type]); in check_unmap()
1002 } else if ((entry->type == dma_debug_coherent) && in check_unmap()
1003 (phys_addr(ref) != phys_addr(entry))) { in check_unmap()
1004 err_printk(ref->dev, entry, "device driver frees " in check_unmap()
1010 phys_addr(entry), in check_unmap()
1015 ref->sg_call_ents != entry->sg_call_ents) { in check_unmap()
1016 err_printk(ref->dev, entry, "device driver frees " in check_unmap()
1017 "DMA sg list with different entry count " in check_unmap()
1019 entry->sg_call_ents, ref->sg_call_ents); in check_unmap()
1026 if (ref->direction != entry->direction) { in check_unmap()
1027 err_printk(ref->dev, entry, "device driver frees " in check_unmap()
1032 dir2name[entry->direction], in check_unmap()
1041 if (entry->map_err_type == MAP_ERR_NOT_CHECKED) { in check_unmap()
1042 err_printk(ref->dev, entry, in check_unmap()
1047 type2name[entry->type]); in check_unmap()
1050 hash_bucket_del(entry); in check_unmap()
1051 dma_entry_free(entry); in check_unmap()
1095 struct dma_debug_entry *entry; in check_sync() local
1101 entry = bucket_find_contain(&bucket, ref, &flags); in check_sync()
1103 if (!entry) { in check_sync()
1111 if (ref->size > entry->size) { in check_sync()
1112 err_printk(dev, entry, "device driver syncs" in check_sync()
1117 entry->dev_addr, entry->size, in check_sync()
1121 if (entry->direction == DMA_BIDIRECTIONAL) in check_sync()
1124 if (ref->direction != entry->direction) { in check_sync()
1125 err_printk(dev, entry, "device driver syncs " in check_sync()
1129 (unsigned long long)ref->dev_addr, entry->size, in check_sync()
1130 dir2name[entry->direction], in check_sync()
1134 if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) && in check_sync()
1136 err_printk(dev, entry, "device driver syncs " in check_sync()
1140 (unsigned long long)ref->dev_addr, entry->size, in check_sync()
1141 dir2name[entry->direction], in check_sync()
1144 if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) && in check_sync()
1146 err_printk(dev, entry, "device driver syncs " in check_sync()
1150 (unsigned long long)ref->dev_addr, entry->size, in check_sync()
1151 dir2name[entry->direction], in check_sync()
1155 ref->sg_call_ents != entry->sg_call_ents) { in check_sync()
1156 err_printk(ref->dev, entry, "device driver syncs " in check_sync()
1157 "DMA sg list with different entry count " in check_sync()
1159 entry->sg_call_ents, ref->sg_call_ents); in check_sync()
1212 struct dma_debug_entry *entry; in debug_dma_map_page() local
1220 entry = dma_entry_alloc(); in debug_dma_map_page()
1221 if (!entry) in debug_dma_map_page()
1224 entry->dev = dev; in debug_dma_map_page()
1225 entry->type = dma_debug_single; in debug_dma_map_page()
1226 entry->pfn = page_to_pfn(page); in debug_dma_map_page()
1227 entry->offset = offset; in debug_dma_map_page()
1228 entry->dev_addr = dma_addr; in debug_dma_map_page()
1229 entry->size = size; in debug_dma_map_page()
1230 entry->direction = direction; in debug_dma_map_page()
1231 entry->map_err_type = MAP_ERR_NOT_CHECKED; in debug_dma_map_page()
1241 add_dma_entry(entry, attrs); in debug_dma_map_page()
1247 struct dma_debug_entry *entry; in debug_dma_mapping_error() local
1258 list_for_each_entry(entry, &bucket->list, list) { in debug_dma_mapping_error()
1259 if (!exact_match(&ref, entry)) in debug_dma_mapping_error()
1268 * best-fit algorithm here which updates the first entry in debug_dma_mapping_error()
1272 if (entry->map_err_type == MAP_ERR_NOT_CHECKED) { in debug_dma_mapping_error()
1273 entry->map_err_type = MAP_ERR_CHECKED; in debug_dma_mapping_error()
1302 struct dma_debug_entry *entry; in debug_dma_map_sg() local
1316 entry = dma_entry_alloc(); in debug_dma_map_sg()
1317 if (!entry) in debug_dma_map_sg()
1320 entry->type = dma_debug_sg; in debug_dma_map_sg()
1321 entry->dev = dev; in debug_dma_map_sg()
1322 entry->pfn = page_to_pfn(sg_page(s)); in debug_dma_map_sg()
1323 entry->offset = s->offset; in debug_dma_map_sg()
1324 entry->size = sg_dma_len(s); in debug_dma_map_sg()
1325 entry->dev_addr = sg_dma_address(s); in debug_dma_map_sg()
1326 entry->direction = direction; in debug_dma_map_sg()
1327 entry->sg_call_ents = nents; in debug_dma_map_sg()
1328 entry->sg_mapped_ents = mapped_ents; in debug_dma_map_sg()
1332 add_dma_entry(entry, attrs); in debug_dma_map_sg()
1339 struct dma_debug_entry *entry; in get_nr_mapped_entries() local
1345 entry = bucket_find_exact(bucket, ref); in get_nr_mapped_entries()
1348 if (entry) in get_nr_mapped_entries()
1349 mapped_ents = entry->sg_mapped_ents; in get_nr_mapped_entries()
1391 struct dma_debug_entry *entry; in debug_dma_alloc_coherent() local
1403 entry = dma_entry_alloc(); in debug_dma_alloc_coherent()
1404 if (!entry) in debug_dma_alloc_coherent()
1407 entry->type = dma_debug_coherent; in debug_dma_alloc_coherent()
1408 entry->dev = dev; in debug_dma_alloc_coherent()
1409 entry->offset = offset_in_page(virt); in debug_dma_alloc_coherent()
1410 entry->size = size; in debug_dma_alloc_coherent()
1411 entry->dev_addr = dma_addr; in debug_dma_alloc_coherent()
1412 entry->direction = DMA_BIDIRECTIONAL; in debug_dma_alloc_coherent()
1415 entry->pfn = vmalloc_to_pfn(virt); in debug_dma_alloc_coherent()
1417 entry->pfn = page_to_pfn(virt_to_page(virt)); in debug_dma_alloc_coherent()
1419 add_dma_entry(entry, attrs); in debug_dma_alloc_coherent()
1453 struct dma_debug_entry *entry; in debug_dma_map_resource() local
1458 entry = dma_entry_alloc(); in debug_dma_map_resource()
1459 if (!entry) in debug_dma_map_resource()
1462 entry->type = dma_debug_resource; in debug_dma_map_resource()
1463 entry->dev = dev; in debug_dma_map_resource()
1464 entry->pfn = PHYS_PFN(addr); in debug_dma_map_resource()
1465 entry->offset = offset_in_page(addr); in debug_dma_map_resource()
1466 entry->size = size; in debug_dma_map_resource()
1467 entry->dev_addr = dma_addr; in debug_dma_map_resource()
1468 entry->direction = direction; in debug_dma_map_resource()
1469 entry->map_err_type = MAP_ERR_NOT_CHECKED; in debug_dma_map_resource()
1471 add_dma_entry(entry, attrs); in debug_dma_map_resource()