Lines Matching refs:zram
54 static void zram_free_page(struct zram *zram, size_t index);
56 static void zram_slot_lock(struct zram *zram, u32 index) in zram_slot_lock() argument
58 bit_spin_lock(ZRAM_LOCK, &zram->table[index].value); in zram_slot_lock()
61 static void zram_slot_unlock(struct zram *zram, u32 index) in zram_slot_unlock() argument
63 bit_spin_unlock(ZRAM_LOCK, &zram->table[index].value); in zram_slot_unlock()
66 static inline bool init_done(struct zram *zram) in init_done() argument
68 return zram->disksize; in init_done()
71 static inline bool zram_allocated(struct zram *zram, u32 index) in zram_allocated() argument
74 return (zram->table[index].value >> (ZRAM_FLAG_SHIFT + 1)) || in zram_allocated()
75 zram->table[index].handle; in zram_allocated()
78 static inline struct zram *dev_to_zram(struct device *dev) in dev_to_zram()
80 return (struct zram *)dev_to_disk(dev)->private_data; in dev_to_zram()
83 static unsigned long zram_get_handle(struct zram *zram, u32 index) in zram_get_handle() argument
85 return zram->table[index].handle; in zram_get_handle()
88 static void zram_set_handle(struct zram *zram, u32 index, unsigned long handle) in zram_set_handle() argument
90 zram->table[index].handle = handle; in zram_set_handle()
94 static bool zram_test_flag(struct zram *zram, u32 index, in zram_test_flag() argument
97 return zram->table[index].value & BIT(flag); in zram_test_flag()
100 static void zram_set_flag(struct zram *zram, u32 index, in zram_set_flag() argument
103 zram->table[index].value |= BIT(flag); in zram_set_flag()
106 static void zram_clear_flag(struct zram *zram, u32 index, in zram_clear_flag() argument
109 zram->table[index].value &= ~BIT(flag); in zram_clear_flag()
112 static inline void zram_set_element(struct zram *zram, u32 index, in zram_set_element() argument
115 zram->table[index].element = element; in zram_set_element()
118 static unsigned long zram_get_element(struct zram *zram, u32 index) in zram_get_element() argument
120 return zram->table[index].element; in zram_get_element()
123 static size_t zram_get_obj_size(struct zram *zram, u32 index) in zram_get_obj_size() argument
125 return zram->table[index].value & (BIT(ZRAM_FLAG_SHIFT) - 1); in zram_get_obj_size()
128 static void zram_set_obj_size(struct zram *zram, in zram_set_obj_size() argument
131 unsigned long flags = zram->table[index].value >> ZRAM_FLAG_SHIFT; in zram_set_obj_size()
133 zram->table[index].value = (flags << ZRAM_FLAG_SHIFT) | size; in zram_set_obj_size()
151 static inline bool valid_io_request(struct zram *zram, in valid_io_request() argument
163 bound = zram->disksize >> SECTOR_SHIFT; in valid_io_request()
178 static inline void update_used_max(struct zram *zram, in update_used_max() argument
183 old_max = atomic_long_read(&zram->stats.max_used_pages); in update_used_max()
189 &zram->stats.max_used_pages, cur_max, pages); in update_used_max()
223 struct zram *zram = dev_to_zram(dev); in initstate_show() local
225 down_read(&zram->init_lock); in initstate_show()
226 val = init_done(zram); in initstate_show()
227 up_read(&zram->init_lock); in initstate_show()
235 struct zram *zram = dev_to_zram(dev); in disksize_show() local
237 return scnprintf(buf, PAGE_SIZE, "%llu\n", zram->disksize); in disksize_show()
245 struct zram *zram = dev_to_zram(dev); in mem_limit_store() local
251 down_write(&zram->init_lock); in mem_limit_store()
252 zram->limit_pages = PAGE_ALIGN(limit) >> PAGE_SHIFT; in mem_limit_store()
253 up_write(&zram->init_lock); in mem_limit_store()
263 struct zram *zram = dev_to_zram(dev); in mem_used_max_store() local
269 down_read(&zram->init_lock); in mem_used_max_store()
270 if (init_done(zram)) { in mem_used_max_store()
271 atomic_long_set(&zram->stats.max_used_pages, in mem_used_max_store()
272 zs_get_total_pages(zram->mem_pool)); in mem_used_max_store()
274 up_read(&zram->init_lock); in mem_used_max_store()
280 static bool zram_wb_enabled(struct zram *zram) in zram_wb_enabled() argument
282 return zram->backing_dev; in zram_wb_enabled()
285 static void reset_bdev(struct zram *zram) in reset_bdev() argument
289 if (!zram_wb_enabled(zram)) in reset_bdev()
292 bdev = zram->bdev; in reset_bdev()
293 if (zram->old_block_size) in reset_bdev()
294 set_blocksize(bdev, zram->old_block_size); in reset_bdev()
297 filp_close(zram->backing_dev, NULL); in reset_bdev()
298 zram->backing_dev = NULL; in reset_bdev()
299 zram->old_block_size = 0; in reset_bdev()
300 zram->bdev = NULL; in reset_bdev()
301 zram->disk->queue->backing_dev_info->capabilities |= in reset_bdev()
303 kvfree(zram->bitmap); in reset_bdev()
304 zram->bitmap = NULL; in reset_bdev()
310 struct zram *zram = dev_to_zram(dev); in backing_dev_show() local
311 struct file *file = zram->backing_dev; in backing_dev_show()
315 down_read(&zram->init_lock); in backing_dev_show()
316 if (!zram_wb_enabled(zram)) { in backing_dev_show()
318 up_read(&zram->init_lock); in backing_dev_show()
332 up_read(&zram->init_lock); in backing_dev_show()
348 struct zram *zram = dev_to_zram(dev); in backing_dev_store() local
354 down_write(&zram->init_lock); in backing_dev_store()
355 if (init_done(zram)) { in backing_dev_store()
384 err = blkdev_get(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL, zram); in backing_dev_store()
401 reset_bdev(zram); in backing_dev_store()
402 spin_lock_init(&zram->bitmap_lock); in backing_dev_store()
404 zram->old_block_size = old_block_size; in backing_dev_store()
405 zram->bdev = bdev; in backing_dev_store()
406 zram->backing_dev = backing_dev; in backing_dev_store()
407 zram->bitmap = bitmap; in backing_dev_store()
408 zram->nr_pages = nr_pages; in backing_dev_store()
419 zram->disk->queue->backing_dev_info->capabilities &= in backing_dev_store()
421 up_write(&zram->init_lock); in backing_dev_store()
437 up_write(&zram->init_lock); in backing_dev_store()
444 static unsigned long get_entry_bdev(struct zram *zram) in get_entry_bdev() argument
448 spin_lock(&zram->bitmap_lock); in get_entry_bdev()
450 entry = find_next_zero_bit(zram->bitmap, zram->nr_pages, 1); in get_entry_bdev()
451 if (entry == zram->nr_pages) { in get_entry_bdev()
452 spin_unlock(&zram->bitmap_lock); in get_entry_bdev()
456 set_bit(entry, zram->bitmap); in get_entry_bdev()
457 spin_unlock(&zram->bitmap_lock); in get_entry_bdev()
462 static void put_entry_bdev(struct zram *zram, unsigned long entry) in put_entry_bdev() argument
466 spin_lock(&zram->bitmap_lock); in put_entry_bdev()
467 was_set = test_and_clear_bit(entry, zram->bitmap); in put_entry_bdev()
468 spin_unlock(&zram->bitmap_lock); in put_entry_bdev()
484 static int read_from_bdev_async(struct zram *zram, struct bio_vec *bvec, in read_from_bdev_async() argument
494 bio_set_dev(bio, zram->bdev); in read_from_bdev_async()
514 struct zram *zram; member
524 struct zram *zram = zw->zram; in zram_sync_read() local
528 read_from_bdev_async(zram, &bvec, entry, bio); in zram_sync_read()
536 static int read_from_bdev_sync(struct zram *zram, struct bio_vec *bvec, in read_from_bdev_sync() argument
541 work.zram = zram; in read_from_bdev_sync()
553 static int read_from_bdev_sync(struct zram *zram, struct bio_vec *bvec, in read_from_bdev_sync() argument
561 static int read_from_bdev(struct zram *zram, struct bio_vec *bvec, in read_from_bdev() argument
565 return read_from_bdev_sync(zram, bvec, entry, parent); in read_from_bdev()
567 return read_from_bdev_async(zram, bvec, entry, parent); in read_from_bdev()
570 static int write_to_bdev(struct zram *zram, struct bio_vec *bvec, in write_to_bdev() argument
581 entry = get_entry_bdev(zram); in write_to_bdev()
588 bio_set_dev(bio, zram->bdev); in write_to_bdev()
592 put_entry_bdev(zram, entry); in write_to_bdev()
610 static void zram_wb_clear(struct zram *zram, u32 index) in zram_wb_clear() argument
614 zram_clear_flag(zram, index, ZRAM_WB); in zram_wb_clear()
615 entry = zram_get_element(zram, index); in zram_wb_clear()
616 zram_set_element(zram, index, 0); in zram_wb_clear()
617 put_entry_bdev(zram, entry); in zram_wb_clear()
621 static bool zram_wb_enabled(struct zram *zram) { return false; } in zram_wb_enabled() argument
622 static inline void reset_bdev(struct zram *zram) {}; in reset_bdev() argument
623 static int write_to_bdev(struct zram *zram, struct bio_vec *bvec, in write_to_bdev() argument
631 static int read_from_bdev(struct zram *zram, struct bio_vec *bvec, in read_from_bdev() argument
636 static void zram_wb_clear(struct zram *zram, u32 index) {} in zram_wb_clear() argument
653 static void zram_accessed(struct zram *zram, u32 index) in zram_accessed() argument
655 zram->table[index].ac_time = ktime_get_boottime(); in zram_accessed()
658 static void zram_reset_access(struct zram *zram, u32 index) in zram_reset_access() argument
660 zram->table[index].ac_time = 0; in zram_reset_access()
668 struct zram *zram = file->private_data; in read_block_state() local
669 unsigned long nr_pages = zram->disksize >> PAGE_SHIFT; in read_block_state()
676 down_read(&zram->init_lock); in read_block_state()
677 if (!init_done(zram)) { in read_block_state()
678 up_read(&zram->init_lock); in read_block_state()
686 zram_slot_lock(zram, index); in read_block_state()
687 if (!zram_allocated(zram, index)) in read_block_state()
690 ts = ktime_to_timespec64(zram->table[index].ac_time); in read_block_state()
695 zram_test_flag(zram, index, ZRAM_SAME) ? 's' : '.', in read_block_state()
696 zram_test_flag(zram, index, ZRAM_WB) ? 'w' : '.', in read_block_state()
697 zram_test_flag(zram, index, ZRAM_HUGE) ? 'h' : '.'); in read_block_state()
700 zram_slot_unlock(zram, index); in read_block_state()
706 zram_slot_unlock(zram, index); in read_block_state()
710 up_read(&zram->init_lock); in read_block_state()
724 static void zram_debugfs_register(struct zram *zram) in zram_debugfs_register() argument
729 zram->debugfs_dir = debugfs_create_dir(zram->disk->disk_name, in zram_debugfs_register()
731 debugfs_create_file("block_state", 0400, zram->debugfs_dir, in zram_debugfs_register()
732 zram, &proc_zram_block_state_op); in zram_debugfs_register()
735 static void zram_debugfs_unregister(struct zram *zram) in zram_debugfs_unregister() argument
737 debugfs_remove_recursive(zram->debugfs_dir); in zram_debugfs_unregister()
742 static void zram_accessed(struct zram *zram, u32 index) {}; in zram_accessed() argument
743 static void zram_reset_access(struct zram *zram, u32 index) {}; in zram_reset_access() argument
744 static void zram_debugfs_register(struct zram *zram) {}; in zram_debugfs_register() argument
745 static void zram_debugfs_unregister(struct zram *zram) {}; in zram_debugfs_unregister() argument
773 struct zram *zram = dev_to_zram(dev); in comp_algorithm_show() local
775 down_read(&zram->init_lock); in comp_algorithm_show()
776 sz = zcomp_available_show(zram->compressor, buf); in comp_algorithm_show()
777 up_read(&zram->init_lock); in comp_algorithm_show()
785 struct zram *zram = dev_to_zram(dev); in comp_algorithm_store() local
786 char compressor[ARRAY_SIZE(zram->compressor)]; in comp_algorithm_store()
798 down_write(&zram->init_lock); in comp_algorithm_store()
799 if (init_done(zram)) { in comp_algorithm_store()
800 up_write(&zram->init_lock); in comp_algorithm_store()
805 strcpy(zram->compressor, compressor); in comp_algorithm_store()
806 up_write(&zram->init_lock); in comp_algorithm_store()
813 struct zram *zram = dev_to_zram(dev); in compact_store() local
815 down_read(&zram->init_lock); in compact_store()
816 if (!init_done(zram)) { in compact_store()
817 up_read(&zram->init_lock); in compact_store()
821 zs_compact(zram->mem_pool); in compact_store()
822 up_read(&zram->init_lock); in compact_store()
830 struct zram *zram = dev_to_zram(dev); in io_stat_show() local
833 down_read(&zram->init_lock); in io_stat_show()
836 (u64)atomic64_read(&zram->stats.failed_reads), in io_stat_show()
837 (u64)atomic64_read(&zram->stats.failed_writes), in io_stat_show()
838 (u64)atomic64_read(&zram->stats.invalid_io), in io_stat_show()
839 (u64)atomic64_read(&zram->stats.notify_free)); in io_stat_show()
840 up_read(&zram->init_lock); in io_stat_show()
848 struct zram *zram = dev_to_zram(dev); in mm_stat_show() local
856 down_read(&zram->init_lock); in mm_stat_show()
857 if (init_done(zram)) { in mm_stat_show()
858 mem_used = zs_get_total_pages(zram->mem_pool); in mm_stat_show()
859 zs_pool_stats(zram->mem_pool, &pool_stats); in mm_stat_show()
862 orig_size = atomic64_read(&zram->stats.pages_stored); in mm_stat_show()
863 max_used = atomic_long_read(&zram->stats.max_used_pages); in mm_stat_show()
868 (u64)atomic64_read(&zram->stats.compr_data_size), in mm_stat_show()
870 zram->limit_pages << PAGE_SHIFT, in mm_stat_show()
872 (u64)atomic64_read(&zram->stats.same_pages), in mm_stat_show()
874 (u64)atomic64_read(&zram->stats.huge_pages)); in mm_stat_show()
875 up_read(&zram->init_lock); in mm_stat_show()
884 struct zram *zram = dev_to_zram(dev); in debug_stat_show() local
887 down_read(&zram->init_lock); in debug_stat_show()
891 (u64)atomic64_read(&zram->stats.writestall)); in debug_stat_show()
892 up_read(&zram->init_lock); in debug_stat_show()
901 static void zram_meta_free(struct zram *zram, u64 disksize) in zram_meta_free() argument
908 zram_free_page(zram, index); in zram_meta_free()
910 zs_destroy_pool(zram->mem_pool); in zram_meta_free()
911 vfree(zram->table); in zram_meta_free()
914 static bool zram_meta_alloc(struct zram *zram, u64 disksize) in zram_meta_alloc() argument
919 zram->table = vzalloc(array_size(num_pages, sizeof(*zram->table))); in zram_meta_alloc()
920 if (!zram->table) in zram_meta_alloc()
923 zram->mem_pool = zs_create_pool(zram->disk->disk_name); in zram_meta_alloc()
924 if (!zram->mem_pool) { in zram_meta_alloc()
925 vfree(zram->table); in zram_meta_alloc()
930 huge_class_size = zs_huge_class_size(zram->mem_pool); in zram_meta_alloc()
939 static void zram_free_page(struct zram *zram, size_t index) in zram_free_page() argument
943 zram_reset_access(zram, index); in zram_free_page()
945 if (zram_test_flag(zram, index, ZRAM_HUGE)) { in zram_free_page()
946 zram_clear_flag(zram, index, ZRAM_HUGE); in zram_free_page()
947 atomic64_dec(&zram->stats.huge_pages); in zram_free_page()
950 if (zram_wb_enabled(zram) && zram_test_flag(zram, index, ZRAM_WB)) { in zram_free_page()
951 zram_wb_clear(zram, index); in zram_free_page()
952 atomic64_dec(&zram->stats.pages_stored); in zram_free_page()
960 if (zram_test_flag(zram, index, ZRAM_SAME)) { in zram_free_page()
961 zram_clear_flag(zram, index, ZRAM_SAME); in zram_free_page()
962 zram_set_element(zram, index, 0); in zram_free_page()
963 atomic64_dec(&zram->stats.same_pages); in zram_free_page()
964 atomic64_dec(&zram->stats.pages_stored); in zram_free_page()
968 handle = zram_get_handle(zram, index); in zram_free_page()
972 zs_free(zram->mem_pool, handle); in zram_free_page()
974 atomic64_sub(zram_get_obj_size(zram, index), in zram_free_page()
975 &zram->stats.compr_data_size); in zram_free_page()
976 atomic64_dec(&zram->stats.pages_stored); in zram_free_page()
978 zram_set_handle(zram, index, 0); in zram_free_page()
979 zram_set_obj_size(zram, index, 0); in zram_free_page()
982 static int __zram_bvec_read(struct zram *zram, struct page *page, u32 index, in __zram_bvec_read() argument
990 if (zram_wb_enabled(zram)) { in __zram_bvec_read()
991 zram_slot_lock(zram, index); in __zram_bvec_read()
992 if (zram_test_flag(zram, index, ZRAM_WB)) { in __zram_bvec_read()
995 zram_slot_unlock(zram, index); in __zram_bvec_read()
1000 return read_from_bdev(zram, &bvec, in __zram_bvec_read()
1001 zram_get_element(zram, index), in __zram_bvec_read()
1004 zram_slot_unlock(zram, index); in __zram_bvec_read()
1007 zram_slot_lock(zram, index); in __zram_bvec_read()
1008 handle = zram_get_handle(zram, index); in __zram_bvec_read()
1009 if (!handle || zram_test_flag(zram, index, ZRAM_SAME)) { in __zram_bvec_read()
1013 value = handle ? zram_get_element(zram, index) : 0; in __zram_bvec_read()
1017 zram_slot_unlock(zram, index); in __zram_bvec_read()
1021 size = zram_get_obj_size(zram, index); in __zram_bvec_read()
1023 src = zs_map_object(zram->mem_pool, handle, ZS_MM_RO); in __zram_bvec_read()
1030 struct zcomp_strm *zstrm = zcomp_stream_get(zram->comp); in __zram_bvec_read()
1035 zcomp_stream_put(zram->comp); in __zram_bvec_read()
1037 zs_unmap_object(zram->mem_pool, handle); in __zram_bvec_read()
1038 zram_slot_unlock(zram, index); in __zram_bvec_read()
1047 static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, in zram_bvec_read() argument
1061 ret = __zram_bvec_read(zram, page, index, bio, is_partial_io(bvec)); in zram_bvec_read()
1080 static int __zram_bvec_write(struct zram *zram, struct bio_vec *bvec, in __zram_bvec_write() argument
1099 atomic64_inc(&zram->stats.same_pages); in __zram_bvec_write()
1105 zstrm = zcomp_stream_get(zram->comp); in __zram_bvec_write()
1111 zcomp_stream_put(zram->comp); in __zram_bvec_write()
1113 zs_free(zram->mem_pool, handle); in __zram_bvec_write()
1119 if (zram_wb_enabled(zram) && allow_wb) { in __zram_bvec_write()
1120 zcomp_stream_put(zram->comp); in __zram_bvec_write()
1121 ret = write_to_bdev(zram, bvec, index, bio, &element); in __zram_bvec_write()
1146 handle = zs_malloc(zram->mem_pool, comp_len, in __zram_bvec_write()
1152 zcomp_stream_put(zram->comp); in __zram_bvec_write()
1153 atomic64_inc(&zram->stats.writestall); in __zram_bvec_write()
1154 handle = zs_malloc(zram->mem_pool, comp_len, in __zram_bvec_write()
1162 alloced_pages = zs_get_total_pages(zram->mem_pool); in __zram_bvec_write()
1163 update_used_max(zram, alloced_pages); in __zram_bvec_write()
1165 if (zram->limit_pages && alloced_pages > zram->limit_pages) { in __zram_bvec_write()
1166 zcomp_stream_put(zram->comp); in __zram_bvec_write()
1167 zs_free(zram->mem_pool, handle); in __zram_bvec_write()
1171 dst = zs_map_object(zram->mem_pool, handle, ZS_MM_WO); in __zram_bvec_write()
1180 zcomp_stream_put(zram->comp); in __zram_bvec_write()
1181 zs_unmap_object(zram->mem_pool, handle); in __zram_bvec_write()
1182 atomic64_add(comp_len, &zram->stats.compr_data_size); in __zram_bvec_write()
1188 zram_slot_lock(zram, index); in __zram_bvec_write()
1189 zram_free_page(zram, index); in __zram_bvec_write()
1192 zram_set_flag(zram, index, ZRAM_HUGE); in __zram_bvec_write()
1193 atomic64_inc(&zram->stats.huge_pages); in __zram_bvec_write()
1197 zram_set_flag(zram, index, flags); in __zram_bvec_write()
1198 zram_set_element(zram, index, element); in __zram_bvec_write()
1200 zram_set_handle(zram, index, handle); in __zram_bvec_write()
1201 zram_set_obj_size(zram, index, comp_len); in __zram_bvec_write()
1203 zram_slot_unlock(zram, index); in __zram_bvec_write()
1206 atomic64_inc(&zram->stats.pages_stored); in __zram_bvec_write()
1210 static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, in zram_bvec_write() argument
1229 ret = __zram_bvec_read(zram, page, index, bio, true); in zram_bvec_write()
1244 ret = __zram_bvec_write(zram, &vec, index, bio); in zram_bvec_write()
1256 static void zram_bio_discard(struct zram *zram, u32 index, in zram_bio_discard() argument
1280 zram_slot_lock(zram, index); in zram_bio_discard()
1281 zram_free_page(zram, index); in zram_bio_discard()
1282 zram_slot_unlock(zram, index); in zram_bio_discard()
1283 atomic64_inc(&zram->stats.notify_free); in zram_bio_discard()
1294 static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index, in zram_bvec_rw() argument
1298 struct request_queue *q = zram->disk->queue; in zram_bvec_rw()
1302 &zram->disk->part0); in zram_bvec_rw()
1305 atomic64_inc(&zram->stats.num_reads); in zram_bvec_rw()
1306 ret = zram_bvec_read(zram, bvec, index, offset, bio); in zram_bvec_rw()
1309 atomic64_inc(&zram->stats.num_writes); in zram_bvec_rw()
1310 ret = zram_bvec_write(zram, bvec, index, offset, bio); in zram_bvec_rw()
1313 generic_end_io_acct(q, op, &zram->disk->part0, start_time); in zram_bvec_rw()
1315 zram_slot_lock(zram, index); in zram_bvec_rw()
1316 zram_accessed(zram, index); in zram_bvec_rw()
1317 zram_slot_unlock(zram, index); in zram_bvec_rw()
1321 atomic64_inc(&zram->stats.failed_reads); in zram_bvec_rw()
1323 atomic64_inc(&zram->stats.failed_writes); in zram_bvec_rw()
1329 static void __zram_make_request(struct zram *zram, struct bio *bio) in __zram_make_request() argument
1343 zram_bio_discard(zram, index, offset, bio); in __zram_make_request()
1357 if (zram_bvec_rw(zram, &bv, index, offset, in __zram_make_request()
1380 struct zram *zram = queue->queuedata; in zram_make_request() local
1382 if (!valid_io_request(zram, bio->bi_iter.bi_sector, in zram_make_request()
1384 atomic64_inc(&zram->stats.invalid_io); in zram_make_request()
1388 __zram_make_request(zram, bio); in zram_make_request()
1399 struct zram *zram; in zram_slot_free_notify() local
1401 zram = bdev->bd_disk->private_data; in zram_slot_free_notify()
1403 zram_slot_lock(zram, index); in zram_slot_free_notify()
1404 zram_free_page(zram, index); in zram_slot_free_notify()
1405 zram_slot_unlock(zram, index); in zram_slot_free_notify()
1406 atomic64_inc(&zram->stats.notify_free); in zram_slot_free_notify()
1414 struct zram *zram; in zram_rw_page() local
1419 zram = bdev->bd_disk->private_data; in zram_rw_page()
1421 if (!valid_io_request(zram, sector, PAGE_SIZE)) { in zram_rw_page()
1422 atomic64_inc(&zram->stats.invalid_io); in zram_rw_page()
1434 ret = zram_bvec_rw(zram, &bv, index, offset, op, NULL); in zram_rw_page()
1460 static void zram_reset_device(struct zram *zram) in zram_reset_device() argument
1465 down_write(&zram->init_lock); in zram_reset_device()
1467 zram->limit_pages = 0; in zram_reset_device()
1469 if (!init_done(zram)) { in zram_reset_device()
1470 up_write(&zram->init_lock); in zram_reset_device()
1474 comp = zram->comp; in zram_reset_device()
1475 disksize = zram->disksize; in zram_reset_device()
1476 zram->disksize = 0; in zram_reset_device()
1478 set_capacity(zram->disk, 0); in zram_reset_device()
1479 part_stat_set_all(&zram->disk->part0, 0); in zram_reset_device()
1481 up_write(&zram->init_lock); in zram_reset_device()
1483 zram_meta_free(zram, disksize); in zram_reset_device()
1484 memset(&zram->stats, 0, sizeof(zram->stats)); in zram_reset_device()
1486 reset_bdev(zram); in zram_reset_device()
1494 struct zram *zram = dev_to_zram(dev); in disksize_store() local
1501 down_write(&zram->init_lock); in disksize_store()
1502 if (init_done(zram)) { in disksize_store()
1509 if (!zram_meta_alloc(zram, disksize)) { in disksize_store()
1514 comp = zcomp_create(zram->compressor); in disksize_store()
1517 zram->compressor); in disksize_store()
1522 zram->comp = comp; in disksize_store()
1523 zram->disksize = disksize; in disksize_store()
1524 set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT); in disksize_store()
1526 revalidate_disk(zram->disk); in disksize_store()
1527 up_write(&zram->init_lock); in disksize_store()
1532 zram_meta_free(zram, disksize); in disksize_store()
1534 up_write(&zram->init_lock); in disksize_store()
1543 struct zram *zram; in reset_store() local
1553 zram = dev_to_zram(dev); in reset_store()
1554 bdev = bdget_disk(zram->disk, 0); in reset_store()
1560 if (bdev->bd_openers || zram->claim) { in reset_store()
1567 zram->claim = true; in reset_store()
1572 zram_reset_device(zram); in reset_store()
1573 revalidate_disk(zram->disk); in reset_store()
1577 zram->claim = false; in reset_store()
1586 struct zram *zram; in zram_open() local
1590 zram = bdev->bd_disk->private_data; in zram_open()
1592 if (zram->claim) in zram_open()
1645 struct zram *zram; in zram_add() local
1649 zram = kzalloc(sizeof(struct zram), GFP_KERNEL); in zram_add()
1650 if (!zram) in zram_add()
1653 ret = idr_alloc(&zram_index_idr, zram, 0, 0, GFP_KERNEL); in zram_add()
1658 init_rwsem(&zram->init_lock); in zram_add()
1671 zram->disk = alloc_disk(1); in zram_add()
1672 if (!zram->disk) { in zram_add()
1679 zram->disk->major = zram_major; in zram_add()
1680 zram->disk->first_minor = device_id; in zram_add()
1681 zram->disk->fops = &zram_devops; in zram_add()
1682 zram->disk->queue = queue; in zram_add()
1683 zram->disk->queue->queuedata = zram; in zram_add()
1684 zram->disk->private_data = zram; in zram_add()
1685 snprintf(zram->disk->disk_name, 16, "zram%d", device_id); in zram_add()
1688 set_capacity(zram->disk, 0); in zram_add()
1690 blk_queue_flag_set(QUEUE_FLAG_NONROT, zram->disk->queue); in zram_add()
1691 blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, zram->disk->queue); in zram_add()
1697 blk_queue_physical_block_size(zram->disk->queue, PAGE_SIZE); in zram_add()
1698 blk_queue_logical_block_size(zram->disk->queue, in zram_add()
1700 blk_queue_io_min(zram->disk->queue, PAGE_SIZE); in zram_add()
1701 blk_queue_io_opt(zram->disk->queue, PAGE_SIZE); in zram_add()
1702 zram->disk->queue->limits.discard_granularity = PAGE_SIZE; in zram_add()
1703 blk_queue_max_discard_sectors(zram->disk->queue, UINT_MAX); in zram_add()
1704 blk_queue_flag_set(QUEUE_FLAG_DISCARD, zram->disk->queue); in zram_add()
1715 blk_queue_max_write_zeroes_sectors(zram->disk->queue, UINT_MAX); in zram_add()
1717 zram->disk->queue->backing_dev_info->capabilities |= in zram_add()
1719 add_disk(zram->disk); in zram_add()
1721 ret = sysfs_create_group(&disk_to_dev(zram->disk)->kobj, in zram_add()
1728 strlcpy(zram->compressor, default_compressor, sizeof(zram->compressor)); in zram_add()
1730 zram_debugfs_register(zram); in zram_add()
1731 pr_info("Added device: %s\n", zram->disk->disk_name); in zram_add()
1735 del_gendisk(zram->disk); in zram_add()
1736 put_disk(zram->disk); in zram_add()
1742 kfree(zram); in zram_add()
1746 static int zram_remove(struct zram *zram) in zram_remove() argument
1750 bdev = bdget_disk(zram->disk, 0); in zram_remove()
1755 if (bdev->bd_openers || zram->claim) { in zram_remove()
1761 zram->claim = true; in zram_remove()
1764 zram_debugfs_unregister(zram); in zram_remove()
1772 sysfs_remove_group(&disk_to_dev(zram->disk)->kobj, in zram_remove()
1777 zram_reset_device(zram); in zram_remove()
1780 pr_info("Removed device: %s\n", zram->disk->disk_name); in zram_remove()
1782 del_gendisk(zram->disk); in zram_remove()
1783 blk_cleanup_queue(zram->disk->queue); in zram_remove()
1784 put_disk(zram->disk); in zram_remove()
1785 kfree(zram); in zram_remove()
1818 struct zram *zram; in hot_remove_store() local
1830 zram = idr_find(&zram_index_idr, dev_id); in hot_remove_store()
1831 if (zram) { in hot_remove_store()
1832 ret = zram_remove(zram); in hot_remove_store()