Lines Matching refs:zhdr
239 static inline void z3fold_page_lock(struct z3fold_header *zhdr) in z3fold_page_lock() argument
241 spin_lock(&zhdr->page_lock); in z3fold_page_lock()
245 static inline int z3fold_page_trylock(struct z3fold_header *zhdr) in z3fold_page_trylock() argument
247 return spin_trylock(&zhdr->page_lock); in z3fold_page_trylock()
251 static inline void z3fold_page_unlock(struct z3fold_header *zhdr) in z3fold_page_unlock() argument
253 spin_unlock(&zhdr->page_lock); in z3fold_page_unlock()
261 struct z3fold_header *zhdr; in __get_z3fold_header() local
271 zhdr = (struct z3fold_header *)(addr & PAGE_MASK); in __get_z3fold_header()
273 locked = z3fold_page_trylock(zhdr); in __get_z3fold_header()
280 zhdr = (struct z3fold_header *)(handle & PAGE_MASK); in __get_z3fold_header()
283 return zhdr; in __get_z3fold_header()
298 static inline void put_z3fold_header(struct z3fold_header *zhdr) in put_z3fold_header() argument
300 struct page *page = virt_to_page(zhdr); in put_z3fold_header()
303 z3fold_page_unlock(zhdr); in put_z3fold_header()
309 struct z3fold_header *zhdr; in free_handle() local
319 zhdr = handle_to_z3fold_header(handle); in free_handle()
323 if (zhdr->slots == slots) { in free_handle()
329 zhdr->foreign_handles--; in free_handle()
402 struct z3fold_header *zhdr = page_address(page); in init_z3fold_page() local
412 return zhdr; in init_z3fold_page()
418 spin_lock_init(&zhdr->page_lock); in init_z3fold_page()
419 kref_init(&zhdr->refcount); in init_z3fold_page()
420 zhdr->first_chunks = 0; in init_z3fold_page()
421 zhdr->middle_chunks = 0; in init_z3fold_page()
422 zhdr->last_chunks = 0; in init_z3fold_page()
423 zhdr->first_num = 0; in init_z3fold_page()
424 zhdr->start_middle = 0; in init_z3fold_page()
425 zhdr->cpu = -1; in init_z3fold_page()
426 zhdr->foreign_handles = 0; in init_z3fold_page()
427 zhdr->mapped_count = 0; in init_z3fold_page()
428 zhdr->slots = slots; in init_z3fold_page()
429 zhdr->pool = pool; in init_z3fold_page()
430 INIT_LIST_HEAD(&zhdr->buddy); in init_z3fold_page()
431 INIT_WORK(&zhdr->work, compact_page_work); in init_z3fold_page()
432 return zhdr; in init_z3fold_page()
448 static inline int __idx(struct z3fold_header *zhdr, enum buddy bud) in __idx() argument
450 return (bud + zhdr->first_num) & BUDDY_MASK; in __idx()
457 static unsigned long __encode_handle(struct z3fold_header *zhdr, in __encode_handle() argument
461 unsigned long h = (unsigned long)zhdr; in __encode_handle()
472 idx = __idx(zhdr, bud); in __encode_handle()
475 h |= (zhdr->last_chunks << BUDDY_SHIFT); in __encode_handle()
483 static unsigned long encode_handle(struct z3fold_header *zhdr, enum buddy bud) in encode_handle() argument
485 return __encode_handle(zhdr, zhdr->slots, bud); in encode_handle()
507 struct z3fold_header *zhdr; in handle_to_buddy() local
515 zhdr = (struct z3fold_header *)(addr & PAGE_MASK); in handle_to_buddy()
516 return (addr - zhdr->first_num) & BUDDY_MASK; in handle_to_buddy()
519 static inline struct z3fold_pool *zhdr_to_pool(struct z3fold_header *zhdr) in zhdr_to_pool() argument
521 return zhdr->pool; in zhdr_to_pool()
524 static void __release_z3fold_page(struct z3fold_header *zhdr, bool locked) in __release_z3fold_page() argument
526 struct page *page = virt_to_page(zhdr); in __release_z3fold_page()
527 struct z3fold_pool *pool = zhdr_to_pool(zhdr); in __release_z3fold_page()
531 WARN_ON(!list_empty(&zhdr->buddy)); in __release_z3fold_page()
540 read_lock(&zhdr->slots->lock); in __release_z3fold_page()
542 if (zhdr->slots->slot[i]) { in __release_z3fold_page()
548 set_bit(HANDLES_ORPHANED, &zhdr->slots->pool); in __release_z3fold_page()
549 read_unlock(&zhdr->slots->lock); in __release_z3fold_page()
552 kmem_cache_free(pool->c_handle, zhdr->slots); in __release_z3fold_page()
555 z3fold_page_unlock(zhdr); in __release_z3fold_page()
558 list_add(&zhdr->buddy, &pool->stale); in __release_z3fold_page()
566 struct z3fold_header *zhdr = container_of(ref, struct z3fold_header, in release_z3fold_page() local
568 __release_z3fold_page(zhdr, false); in release_z3fold_page()
573 struct z3fold_header *zhdr = container_of(ref, struct z3fold_header, in release_z3fold_page_locked() local
575 WARN_ON(z3fold_page_trylock(zhdr)); in release_z3fold_page_locked()
576 __release_z3fold_page(zhdr, true); in release_z3fold_page_locked()
581 struct z3fold_header *zhdr = container_of(ref, struct z3fold_header, in release_z3fold_page_locked_list() local
583 struct z3fold_pool *pool = zhdr_to_pool(zhdr); in release_z3fold_page_locked_list()
586 list_del_init(&zhdr->buddy); in release_z3fold_page_locked_list()
589 WARN_ON(z3fold_page_trylock(zhdr)); in release_z3fold_page_locked_list()
590 __release_z3fold_page(zhdr, true); in release_z3fold_page_locked_list()
599 struct z3fold_header *zhdr = list_first_entry(&pool->stale, in free_pages_work() local
601 struct page *page = virt_to_page(zhdr); in free_pages_work()
603 list_del(&zhdr->buddy); in free_pages_work()
607 cancel_work_sync(&zhdr->work); in free_pages_work()
619 static int num_free_chunks(struct z3fold_header *zhdr) in num_free_chunks() argument
627 if (zhdr->middle_chunks != 0) { in num_free_chunks()
628 int nfree_before = zhdr->first_chunks ? in num_free_chunks()
629 0 : zhdr->start_middle - ZHDR_CHUNKS; in num_free_chunks()
630 int nfree_after = zhdr->last_chunks ? in num_free_chunks()
632 (zhdr->start_middle + zhdr->middle_chunks); in num_free_chunks()
635 nfree = NCHUNKS - zhdr->first_chunks - zhdr->last_chunks; in num_free_chunks()
641 struct z3fold_header *zhdr) in add_to_unbuddied() argument
643 if (zhdr->first_chunks == 0 || zhdr->last_chunks == 0 || in add_to_unbuddied()
644 zhdr->middle_chunks == 0) { in add_to_unbuddied()
647 int freechunks = num_free_chunks(zhdr); in add_to_unbuddied()
649 list_add(&zhdr->buddy, &unbuddied[freechunks]); in add_to_unbuddied()
651 zhdr->cpu = smp_processor_id(); in add_to_unbuddied()
656 static inline void *mchunk_memmove(struct z3fold_header *zhdr, in mchunk_memmove() argument
659 void *beg = zhdr; in mchunk_memmove()
661 beg + (zhdr->start_middle << CHUNK_SHIFT), in mchunk_memmove()
662 zhdr->middle_chunks << CHUNK_SHIFT); in mchunk_memmove()
665 static inline bool buddy_single(struct z3fold_header *zhdr) in buddy_single() argument
667 return !((zhdr->first_chunks && zhdr->middle_chunks) || in buddy_single()
668 (zhdr->first_chunks && zhdr->last_chunks) || in buddy_single()
669 (zhdr->middle_chunks && zhdr->last_chunks)); in buddy_single()
672 static struct z3fold_header *compact_single_buddy(struct z3fold_header *zhdr) in compact_single_buddy() argument
674 struct z3fold_pool *pool = zhdr_to_pool(zhdr); in compact_single_buddy()
675 void *p = zhdr; in compact_single_buddy()
679 int first_idx = __idx(zhdr, FIRST); in compact_single_buddy()
680 int middle_idx = __idx(zhdr, MIDDLE); in compact_single_buddy()
681 int last_idx = __idx(zhdr, LAST); in compact_single_buddy()
688 if (zhdr->first_chunks && zhdr->slots->slot[first_idx]) { in compact_single_buddy()
690 sz = zhdr->first_chunks << CHUNK_SHIFT; in compact_single_buddy()
691 old_handle = (unsigned long)&zhdr->slots->slot[first_idx]; in compact_single_buddy()
692 moved_chunks = &zhdr->first_chunks; in compact_single_buddy()
693 } else if (zhdr->middle_chunks && zhdr->slots->slot[middle_idx]) { in compact_single_buddy()
694 p += zhdr->start_middle << CHUNK_SHIFT; in compact_single_buddy()
695 sz = zhdr->middle_chunks << CHUNK_SHIFT; in compact_single_buddy()
696 old_handle = (unsigned long)&zhdr->slots->slot[middle_idx]; in compact_single_buddy()
697 moved_chunks = &zhdr->middle_chunks; in compact_single_buddy()
698 } else if (zhdr->last_chunks && zhdr->slots->slot[last_idx]) { in compact_single_buddy()
699 p += PAGE_SIZE - (zhdr->last_chunks << CHUNK_SHIFT); in compact_single_buddy()
700 sz = zhdr->last_chunks << CHUNK_SHIFT; in compact_single_buddy()
701 old_handle = (unsigned long)&zhdr->slots->slot[last_idx]; in compact_single_buddy()
702 moved_chunks = &zhdr->last_chunks; in compact_single_buddy()
714 if (WARN_ON(new_zhdr == zhdr)) in compact_single_buddy()
750 write_lock(&zhdr->slots->lock); in compact_single_buddy()
756 write_unlock(&zhdr->slots->lock); in compact_single_buddy()
780 static int z3fold_compact_page(struct z3fold_header *zhdr) in z3fold_compact_page() argument
782 struct page *page = virt_to_page(zhdr); in z3fold_compact_page()
790 if (zhdr->middle_chunks == 0) in z3fold_compact_page()
793 if (zhdr->first_chunks == 0 && zhdr->last_chunks == 0) { in z3fold_compact_page()
795 mchunk_memmove(zhdr, ZHDR_CHUNKS); in z3fold_compact_page()
796 zhdr->first_chunks = zhdr->middle_chunks; in z3fold_compact_page()
797 zhdr->middle_chunks = 0; in z3fold_compact_page()
798 zhdr->start_middle = 0; in z3fold_compact_page()
799 zhdr->first_num++; in z3fold_compact_page()
807 if (zhdr->first_chunks != 0 && zhdr->last_chunks == 0 && in z3fold_compact_page()
808 zhdr->start_middle - (zhdr->first_chunks + ZHDR_CHUNKS) >= in z3fold_compact_page()
810 mchunk_memmove(zhdr, zhdr->first_chunks + ZHDR_CHUNKS); in z3fold_compact_page()
811 zhdr->start_middle = zhdr->first_chunks + ZHDR_CHUNKS; in z3fold_compact_page()
813 } else if (zhdr->last_chunks != 0 && zhdr->first_chunks == 0 && in z3fold_compact_page()
814 TOTAL_CHUNKS - (zhdr->last_chunks + zhdr->start_middle in z3fold_compact_page()
815 + zhdr->middle_chunks) >= in z3fold_compact_page()
817 unsigned short new_start = TOTAL_CHUNKS - zhdr->last_chunks - in z3fold_compact_page()
818 zhdr->middle_chunks; in z3fold_compact_page()
819 mchunk_memmove(zhdr, new_start); in z3fold_compact_page()
820 zhdr->start_middle = new_start; in z3fold_compact_page()
827 static void do_compact_page(struct z3fold_header *zhdr, bool locked) in do_compact_page() argument
829 struct z3fold_pool *pool = zhdr_to_pool(zhdr); in do_compact_page()
832 page = virt_to_page(zhdr); in do_compact_page()
834 WARN_ON(z3fold_page_trylock(zhdr)); in do_compact_page()
836 z3fold_page_lock(zhdr); in do_compact_page()
838 z3fold_page_unlock(zhdr); in do_compact_page()
842 list_del_init(&zhdr->buddy); in do_compact_page()
845 if (kref_put(&zhdr->refcount, release_z3fold_page_locked)) { in do_compact_page()
853 z3fold_page_unlock(zhdr); in do_compact_page()
857 if (!zhdr->foreign_handles && buddy_single(zhdr) && in do_compact_page()
858 zhdr->mapped_count == 0 && compact_single_buddy(zhdr)) { in do_compact_page()
859 if (kref_put(&zhdr->refcount, release_z3fold_page_locked)) in do_compact_page()
862 z3fold_page_unlock(zhdr); in do_compact_page()
866 z3fold_compact_page(zhdr); in do_compact_page()
867 add_to_unbuddied(pool, zhdr); in do_compact_page()
868 z3fold_page_unlock(zhdr); in do_compact_page()
873 struct z3fold_header *zhdr = container_of(w, struct z3fold_header, in compact_page_work() local
876 do_compact_page(zhdr, false); in compact_page_work()
883 struct z3fold_header *zhdr = NULL; in __z3fold_alloc() local
894 zhdr = list_first_entry_or_null(READ_ONCE(l), in __z3fold_alloc()
897 if (!zhdr) in __z3fold_alloc()
903 if (unlikely(zhdr != list_first_entry(READ_ONCE(l), in __z3fold_alloc()
905 !z3fold_page_trylock(zhdr)) { in __z3fold_alloc()
907 zhdr = NULL; in __z3fold_alloc()
913 list_del_init(&zhdr->buddy); in __z3fold_alloc()
914 zhdr->cpu = -1; in __z3fold_alloc()
917 page = virt_to_page(zhdr); in __z3fold_alloc()
920 z3fold_page_unlock(zhdr); in __z3fold_alloc()
921 zhdr = NULL; in __z3fold_alloc()
934 kref_get(&zhdr->refcount); in __z3fold_alloc()
939 if (!zhdr) { in __z3fold_alloc()
950 zhdr = list_first_entry_or_null(READ_ONCE(l), in __z3fold_alloc()
953 if (!zhdr || !z3fold_page_trylock(zhdr)) { in __z3fold_alloc()
955 zhdr = NULL; in __z3fold_alloc()
958 list_del_init(&zhdr->buddy); in __z3fold_alloc()
959 zhdr->cpu = -1; in __z3fold_alloc()
962 page = virt_to_page(zhdr); in __z3fold_alloc()
965 z3fold_page_unlock(zhdr); in __z3fold_alloc()
966 zhdr = NULL; in __z3fold_alloc()
971 kref_get(&zhdr->refcount); in __z3fold_alloc()
976 return zhdr; in __z3fold_alloc()
1095 struct z3fold_header *zhdr = NULL; in z3fold_alloc() local
1110 zhdr = __z3fold_alloc(pool, size, can_sleep); in z3fold_alloc()
1111 if (zhdr) { in z3fold_alloc()
1112 if (zhdr->first_chunks == 0) { in z3fold_alloc()
1113 if (zhdr->middle_chunks != 0 && in z3fold_alloc()
1114 chunks >= zhdr->start_middle) in z3fold_alloc()
1118 } else if (zhdr->last_chunks == 0) in z3fold_alloc()
1120 else if (zhdr->middle_chunks == 0) in z3fold_alloc()
1123 if (kref_put(&zhdr->refcount, in z3fold_alloc()
1127 z3fold_page_unlock(zhdr); in z3fold_alloc()
1132 page = virt_to_page(zhdr); in z3fold_alloc()
1141 zhdr = list_first_entry_or_null(&pool->stale, in z3fold_alloc()
1148 if (zhdr) { in z3fold_alloc()
1149 list_del(&zhdr->buddy); in z3fold_alloc()
1151 cancel_work_sync(&zhdr->work); in z3fold_alloc()
1152 page = virt_to_page(zhdr); in z3fold_alloc()
1163 zhdr = init_z3fold_page(page, bud == HEADLESS, pool, gfp); in z3fold_alloc()
1164 if (!zhdr) { in z3fold_alloc()
1184 z3fold_page_lock(zhdr); in z3fold_alloc()
1188 zhdr->first_chunks = chunks; in z3fold_alloc()
1190 zhdr->last_chunks = chunks; in z3fold_alloc()
1192 zhdr->middle_chunks = chunks; in z3fold_alloc()
1193 zhdr->start_middle = zhdr->first_chunks + ZHDR_CHUNKS; in z3fold_alloc()
1195 add_to_unbuddied(pool, zhdr); in z3fold_alloc()
1205 *handle = encode_handle(zhdr, bud); in z3fold_alloc()
1208 z3fold_page_unlock(zhdr); in z3fold_alloc()
1225 struct z3fold_header *zhdr; in z3fold_free() local
1230 zhdr = get_z3fold_header(handle); in z3fold_free()
1231 page = virt_to_page(zhdr); in z3fold_free()
1244 put_z3fold_header(zhdr); in z3fold_free()
1256 zhdr->first_chunks = 0; in z3fold_free()
1259 zhdr->middle_chunks = 0; in z3fold_free()
1262 zhdr->last_chunks = 0; in z3fold_free()
1267 put_z3fold_header(zhdr); in z3fold_free()
1274 if (kref_put(&zhdr->refcount, release_z3fold_page_locked_list)) { in z3fold_free()
1280 z3fold_page_unlock(zhdr); in z3fold_free()
1285 put_z3fold_header(zhdr); in z3fold_free()
1289 if (zhdr->cpu < 0 || !cpu_online(zhdr->cpu)) { in z3fold_free()
1291 list_del_init(&zhdr->buddy); in z3fold_free()
1293 zhdr->cpu = -1; in z3fold_free()
1294 kref_get(&zhdr->refcount); in z3fold_free()
1296 do_compact_page(zhdr, true); in z3fold_free()
1299 kref_get(&zhdr->refcount); in z3fold_free()
1301 queue_work_on(zhdr->cpu, pool->compact_wq, &zhdr->work); in z3fold_free()
1302 put_z3fold_header(zhdr); in z3fold_free()
1344 struct z3fold_header *zhdr = NULL; in z3fold_reclaim_page() local
1375 zhdr = page_address(page); in z3fold_reclaim_page()
1379 if (!z3fold_page_trylock(zhdr)) { in z3fold_reclaim_page()
1381 zhdr = NULL; in z3fold_reclaim_page()
1384 if (zhdr->foreign_handles) { in z3fold_reclaim_page()
1386 z3fold_page_unlock(zhdr); in z3fold_reclaim_page()
1387 zhdr = NULL; in z3fold_reclaim_page()
1390 kref_get(&zhdr->refcount); in z3fold_reclaim_page()
1391 list_del_init(&zhdr->buddy); in z3fold_reclaim_page()
1392 zhdr->cpu = -1; in z3fold_reclaim_page()
1396 if (!zhdr) in z3fold_reclaim_page()
1412 if (zhdr->first_chunks) in z3fold_reclaim_page()
1413 first_handle = encode_handle(zhdr, FIRST); in z3fold_reclaim_page()
1414 if (zhdr->middle_chunks) in z3fold_reclaim_page()
1415 middle_handle = encode_handle(zhdr, MIDDLE); in z3fold_reclaim_page()
1416 if (zhdr->last_chunks) in z3fold_reclaim_page()
1417 last_handle = encode_handle(zhdr, LAST); in z3fold_reclaim_page()
1422 z3fold_page_unlock(zhdr); in z3fold_reclaim_page()
1424 first_handle = encode_handle(zhdr, HEADLESS); in z3fold_reclaim_page()
1458 z3fold_page_lock(zhdr); in z3fold_reclaim_page()
1459 if (kref_put(&zhdr->refcount, in z3fold_reclaim_page()
1472 z3fold_page_unlock(zhdr); in z3fold_reclaim_page()
1495 struct z3fold_header *zhdr; in z3fold_map() local
1500 zhdr = get_z3fold_header(handle); in z3fold_map()
1501 addr = zhdr; in z3fold_map()
1502 page = virt_to_page(zhdr); in z3fold_map()
1513 addr += zhdr->start_middle << CHUNK_SHIFT; in z3fold_map()
1527 zhdr->mapped_count++; in z3fold_map()
1529 put_z3fold_header(zhdr); in z3fold_map()
1540 struct z3fold_header *zhdr; in z3fold_unmap() local
1544 zhdr = get_z3fold_header(handle); in z3fold_unmap()
1545 page = virt_to_page(zhdr); in z3fold_unmap()
1553 zhdr->mapped_count--; in z3fold_unmap()
1554 put_z3fold_header(zhdr); in z3fold_unmap()
1570 struct z3fold_header *zhdr; in z3fold_page_isolate() local
1580 zhdr = page_address(page); in z3fold_page_isolate()
1581 z3fold_page_lock(zhdr); in z3fold_page_isolate()
1586 if (zhdr->mapped_count != 0 || zhdr->foreign_handles != 0) in z3fold_page_isolate()
1589 pool = zhdr_to_pool(zhdr); in z3fold_page_isolate()
1591 if (!list_empty(&zhdr->buddy)) in z3fold_page_isolate()
1592 list_del_init(&zhdr->buddy); in z3fold_page_isolate()
1597 kref_get(&zhdr->refcount); in z3fold_page_isolate()
1598 z3fold_page_unlock(zhdr); in z3fold_page_isolate()
1602 z3fold_page_unlock(zhdr); in z3fold_page_isolate()
1609 struct z3fold_header *zhdr, *new_zhdr; in z3fold_page_migrate() local
1617 zhdr = page_address(page); in z3fold_page_migrate()
1618 pool = zhdr_to_pool(zhdr); in z3fold_page_migrate()
1620 if (!z3fold_page_trylock(zhdr)) { in z3fold_page_migrate()
1623 if (zhdr->mapped_count != 0 || zhdr->foreign_handles != 0) { in z3fold_page_migrate()
1624 z3fold_page_unlock(zhdr); in z3fold_page_migrate()
1627 if (work_pending(&zhdr->work)) { in z3fold_page_migrate()
1628 z3fold_page_unlock(zhdr); in z3fold_page_migrate()
1632 memcpy(new_zhdr, zhdr, PAGE_SIZE); in z3fold_page_migrate()
1635 z3fold_page_unlock(zhdr); in z3fold_page_migrate()
1672 struct z3fold_header *zhdr; in z3fold_page_putback() local
1675 zhdr = page_address(page); in z3fold_page_putback()
1676 pool = zhdr_to_pool(zhdr); in z3fold_page_putback()
1678 z3fold_page_lock(zhdr); in z3fold_page_putback()
1679 if (!list_empty(&zhdr->buddy)) in z3fold_page_putback()
1680 list_del_init(&zhdr->buddy); in z3fold_page_putback()
1682 if (kref_put(&zhdr->refcount, release_z3fold_page_locked)) { in z3fold_page_putback()
1689 z3fold_page_unlock(zhdr); in z3fold_page_putback()