Lines Matching refs:mem
228 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool; in swiotlb_print_info() local
230 if (!mem->nslabs) { in swiotlb_print_info()
235 pr_info("mapped [mem %pa-%pa] (%luMB)\n", &mem->start, &mem->end, in swiotlb_print_info()
236 (mem->nslabs << IO_TLB_SHIFT) >> 20); in swiotlb_print_info()
257 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool; in swiotlb_update_mem_attributes() local
260 if (!mem->nslabs || mem->late_alloc) in swiotlb_update_mem_attributes()
262 bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT); in swiotlb_update_mem_attributes()
263 set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT); in swiotlb_update_mem_attributes()
266 static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start, in swiotlb_init_io_tlb_pool() argument
272 mem->nslabs = nslabs; in swiotlb_init_io_tlb_pool()
273 mem->start = start; in swiotlb_init_io_tlb_pool()
274 mem->end = mem->start + bytes; in swiotlb_init_io_tlb_pool()
275 mem->late_alloc = late_alloc; in swiotlb_init_io_tlb_pool()
276 mem->nareas = nareas; in swiotlb_init_io_tlb_pool()
277 mem->area_nslabs = nslabs / mem->nareas; in swiotlb_init_io_tlb_pool()
279 for (i = 0; i < mem->nareas; i++) { in swiotlb_init_io_tlb_pool()
280 spin_lock_init(&mem->areas[i].lock); in swiotlb_init_io_tlb_pool()
281 mem->areas[i].index = 0; in swiotlb_init_io_tlb_pool()
282 mem->areas[i].used = 0; in swiotlb_init_io_tlb_pool()
285 for (i = 0; i < mem->nslabs; i++) { in swiotlb_init_io_tlb_pool()
286 mem->slots[i].list = IO_TLB_SEGSIZE - io_tlb_offset(i); in swiotlb_init_io_tlb_pool()
287 mem->slots[i].orig_addr = INVALID_PHYS_ADDR; in swiotlb_init_io_tlb_pool()
288 mem->slots[i].alloc_size = 0; in swiotlb_init_io_tlb_pool()
292 mem->vaddr = vaddr; in swiotlb_init_io_tlb_pool()
301 static void add_mem_pool(struct io_tlb_mem *mem, struct io_tlb_pool *pool) in add_mem_pool() argument
304 spin_lock(&mem->lock); in add_mem_pool()
305 list_add_rcu(&pool->node, &mem->pools); in add_mem_pool()
306 mem->nslabs += pool->nslabs; in add_mem_pool()
307 spin_unlock(&mem->lock); in add_mem_pool()
309 mem->nslabs = pool->nslabs; in add_mem_pool()
352 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool; in swiotlb_init_remap() local
393 alloc_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), nslabs)); in swiotlb_init_remap()
394 mem->slots = memblock_alloc(alloc_size, PAGE_SIZE); in swiotlb_init_remap()
395 if (!mem->slots) { in swiotlb_init_remap()
401 mem->areas = memblock_alloc(array_size(sizeof(struct io_tlb_area), in swiotlb_init_remap()
403 if (!mem->areas) { in swiotlb_init_remap()
408 swiotlb_init_io_tlb_pool(mem, __pa(tlb), nslabs, false, nareas); in swiotlb_init_remap()
409 add_mem_pool(&io_tlb_default_mem, mem); in swiotlb_init_remap()
428 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool; in swiotlb_init_late() local
493 area_order = get_order(array_size(sizeof(*mem->areas), nareas)); in swiotlb_init_late()
494 mem->areas = (struct io_tlb_area *) in swiotlb_init_late()
496 if (!mem->areas) in swiotlb_init_late()
499 mem->slots = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, in swiotlb_init_late()
500 get_order(array_size(sizeof(*mem->slots), nslabs))); in swiotlb_init_late()
501 if (!mem->slots) in swiotlb_init_late()
506 swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), nslabs, true, in swiotlb_init_late()
508 add_mem_pool(&io_tlb_default_mem, mem); in swiotlb_init_late()
514 free_pages((unsigned long)mem->areas, area_order); in swiotlb_init_late()
522 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool; in swiotlb_exit() local
530 if (!mem->nslabs) in swiotlb_exit()
534 tbl_vaddr = (unsigned long)phys_to_virt(mem->start); in swiotlb_exit()
535 tbl_size = PAGE_ALIGN(mem->end - mem->start); in swiotlb_exit()
536 slots_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), mem->nslabs)); in swiotlb_exit()
539 if (mem->late_alloc) { in swiotlb_exit()
540 area_order = get_order(array_size(sizeof(*mem->areas), in swiotlb_exit()
541 mem->nareas)); in swiotlb_exit()
542 free_pages((unsigned long)mem->areas, area_order); in swiotlb_exit()
544 free_pages((unsigned long)mem->slots, get_order(slots_size)); in swiotlb_exit()
546 memblock_free_late(__pa(mem->areas), in swiotlb_exit()
547 array_size(sizeof(*mem->areas), mem->nareas)); in swiotlb_exit()
548 memblock_free_late(mem->start, tbl_size); in swiotlb_exit()
549 memblock_free_late(__pa(mem->slots), slots_size); in swiotlb_exit()
552 memset(mem, 0, sizeof(*mem)); in swiotlb_exit()
724 struct io_tlb_mem *mem = in swiotlb_dyn_alloc() local
729 default_nareas, mem->phys_limit, GFP_KERNEL); in swiotlb_dyn_alloc()
735 add_mem_pool(mem, pool); in swiotlb_dyn_alloc()
765 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; in swiotlb_find_pool() local
769 list_for_each_entry_rcu(pool, &mem->pools, node) { in swiotlb_find_pool()
830 struct io_tlb_pool *mem = swiotlb_find_pool(dev, tlb_addr); in swiotlb_bounce() local
831 int index = (tlb_addr - mem->start) >> IO_TLB_SHIFT; in swiotlb_bounce()
832 phys_addr_t orig_addr = mem->slots[index].orig_addr; in swiotlb_bounce()
833 size_t alloc_size = mem->slots[index].alloc_size; in swiotlb_bounce()
835 unsigned char *vaddr = mem->vaddr + tlb_addr - mem->start; in swiotlb_bounce()
910 static unsigned int wrap_area_index(struct io_tlb_pool *mem, unsigned int index) in wrap_area_index() argument
912 if (index >= mem->area_nslabs) in wrap_area_index()
924 static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots) in inc_used_and_hiwater() argument
928 new_used = atomic_long_add_return(nslots, &mem->total_used); in inc_used_and_hiwater()
929 old_hiwater = atomic_long_read(&mem->used_hiwater); in inc_used_and_hiwater()
933 } while (!atomic_long_try_cmpxchg(&mem->used_hiwater, in inc_used_and_hiwater()
937 static void dec_used(struct io_tlb_mem *mem, unsigned int nslots) in dec_used() argument
939 atomic_long_sub(nslots, &mem->total_used); in dec_used()
943 static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots) in inc_used_and_hiwater() argument
946 static void dec_used(struct io_tlb_mem *mem, unsigned int nslots) in dec_used() argument
1114 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; in swiotlb_find_slots() local
1122 list_for_each_entry_rcu(pool, &mem->pools, node) { in swiotlb_find_slots()
1131 if (!mem->can_grow) in swiotlb_find_slots()
1134 schedule_work(&mem->dyn_alloc); in swiotlb_find_slots()
1205 static unsigned long mem_used(struct io_tlb_mem *mem) in mem_used() argument
1207 return atomic_long_read(&mem->total_used); in mem_used()
1239 static unsigned long mem_used(struct io_tlb_mem *mem) in mem_used() argument
1246 list_for_each_entry_rcu(pool, &mem->pools, node) in mem_used()
1252 return mem_pool_used(&mem->defpool); in mem_used()
1263 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; in swiotlb_tbl_map_single() local
1270 if (!mem || !mem->nslabs) { in swiotlb_tbl_map_single()
1291 alloc_size, mem->nslabs, mem_used(mem)); in swiotlb_tbl_map_single()
1316 struct io_tlb_pool *mem = swiotlb_find_pool(dev, tlb_addr); in swiotlb_release_slots() local
1319 int index = (tlb_addr - offset - mem->start) >> IO_TLB_SHIFT; in swiotlb_release_slots()
1320 int nslots = nr_slots(mem->slots[index].alloc_size + offset); in swiotlb_release_slots()
1321 int aindex = index / mem->area_nslabs; in swiotlb_release_slots()
1322 struct io_tlb_area *area = &mem->areas[aindex]; in swiotlb_release_slots()
1331 BUG_ON(aindex >= mem->nareas); in swiotlb_release_slots()
1335 count = mem->slots[index + nslots].list; in swiotlb_release_slots()
1344 mem->slots[i].list = ++count; in swiotlb_release_slots()
1345 mem->slots[i].orig_addr = INVALID_PHYS_ADDR; in swiotlb_release_slots()
1346 mem->slots[i].alloc_size = 0; in swiotlb_release_slots()
1354 io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 && mem->slots[i].list; in swiotlb_release_slots()
1356 mem->slots[i].list = ++count; in swiotlb_release_slots()
1494 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; in is_swiotlb_active() local
1496 return mem && mem->nslabs; in is_swiotlb_active()
1530 struct io_tlb_mem *mem = data; in io_tlb_used_get() local
1532 *val = mem_used(mem); in io_tlb_used_get()
1538 struct io_tlb_mem *mem = data; in io_tlb_hiwater_get() local
1540 *val = atomic_long_read(&mem->used_hiwater); in io_tlb_hiwater_get()
1546 struct io_tlb_mem *mem = data; in io_tlb_hiwater_set() local
1552 atomic_long_set(&mem->used_hiwater, val); in io_tlb_hiwater_set()
1560 static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem, in swiotlb_create_debugfs_files() argument
1563 atomic_long_set(&mem->total_used, 0); in swiotlb_create_debugfs_files()
1564 atomic_long_set(&mem->used_hiwater, 0); in swiotlb_create_debugfs_files()
1566 mem->debugfs = debugfs_create_dir(dirname, io_tlb_default_mem.debugfs); in swiotlb_create_debugfs_files()
1567 if (!mem->nslabs) in swiotlb_create_debugfs_files()
1570 debugfs_create_ulong("io_tlb_nslabs", 0400, mem->debugfs, &mem->nslabs); in swiotlb_create_debugfs_files()
1571 debugfs_create_file("io_tlb_used", 0400, mem->debugfs, mem, in swiotlb_create_debugfs_files()
1573 debugfs_create_file("io_tlb_used_hiwater", 0600, mem->debugfs, mem, in swiotlb_create_debugfs_files()
1587 static inline void swiotlb_create_debugfs_files(struct io_tlb_mem *mem, in swiotlb_create_debugfs_files() argument
1598 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; in swiotlb_alloc() local
1603 if (!mem) in swiotlb_alloc()
1630 struct io_tlb_mem *mem = rmem->priv; in rmem_swiotlb_device_init() local
1646 if (!mem) { in rmem_swiotlb_device_init()
1649 mem = kzalloc(sizeof(*mem), GFP_KERNEL); in rmem_swiotlb_device_init()
1650 if (!mem) in rmem_swiotlb_device_init()
1652 pool = &mem->defpool; in rmem_swiotlb_device_init()
1656 kfree(mem); in rmem_swiotlb_device_init()
1664 kfree(mem); in rmem_swiotlb_device_init()
1672 mem->force_bounce = true; in rmem_swiotlb_device_init()
1673 mem->for_alloc = true; in rmem_swiotlb_device_init()
1675 spin_lock_init(&mem->lock); in rmem_swiotlb_device_init()
1677 add_mem_pool(mem, pool); in rmem_swiotlb_device_init()
1679 rmem->priv = mem; in rmem_swiotlb_device_init()
1681 swiotlb_create_debugfs_files(mem, rmem->name); in rmem_swiotlb_device_init()
1684 dev->dma_io_tlb_mem = mem; in rmem_swiotlb_device_init()