Lines Matching refs:clone
69 struct clone { struct
153 static enum clone_metadata_mode get_clone_mode(struct clone *clone) in get_clone_mode() argument
155 return READ_ONCE(clone->mode); in get_clone_mode()
158 static const char *clone_device_name(struct clone *clone) in clone_device_name() argument
160 return dm_table_device_name(clone->ti->table); in clone_device_name()
163 static void __set_clone_mode(struct clone *clone, enum clone_metadata_mode new_mode) in __set_clone_mode() argument
171 enum clone_metadata_mode old_mode = get_clone_mode(clone); in __set_clone_mode()
180 dm_clone_metadata_set_read_only(clone->cmd); in __set_clone_mode()
184 dm_clone_metadata_set_read_write(clone->cmd); in __set_clone_mode()
188 WRITE_ONCE(clone->mode, new_mode); in __set_clone_mode()
191 dm_table_event(clone->ti->table); in __set_clone_mode()
192 DMINFO("%s: Switching to %s mode", clone_device_name(clone), in __set_clone_mode()
197 static void __abort_transaction(struct clone *clone) in __abort_transaction() argument
199 const char *dev_name = clone_device_name(clone); in __abort_transaction()
201 if (get_clone_mode(clone) >= CM_READ_ONLY) in __abort_transaction()
205 if (dm_clone_metadata_abort(clone->cmd)) { in __abort_transaction()
207 __set_clone_mode(clone, CM_FAIL); in __abort_transaction()
211 static void __reload_in_core_bitset(struct clone *clone) in __reload_in_core_bitset() argument
213 const char *dev_name = clone_device_name(clone); in __reload_in_core_bitset()
215 if (get_clone_mode(clone) == CM_FAIL) in __reload_in_core_bitset()
220 if (dm_clone_reload_in_core_bitset(clone->cmd)) { in __reload_in_core_bitset()
222 __set_clone_mode(clone, CM_FAIL); in __reload_in_core_bitset()
226 static void __metadata_operation_failed(struct clone *clone, const char *op, int r) in __metadata_operation_failed() argument
229 clone_device_name(clone), op, r); in __metadata_operation_failed()
231 __abort_transaction(clone); in __metadata_operation_failed()
232 __set_clone_mode(clone, CM_READ_ONLY); in __metadata_operation_failed()
239 __reload_in_core_bitset(clone); in __metadata_operation_failed()
245 static inline void wakeup_hydration_waiters(struct clone *clone) in wakeup_hydration_waiters() argument
247 wake_up_all(&clone->hydration_stopped); in wakeup_hydration_waiters()
250 static inline void wake_worker(struct clone *clone) in wake_worker() argument
252 queue_work(clone->wq, &clone->worker); in wake_worker()
260 static inline void remap_to_source(struct clone *clone, struct bio *bio) in remap_to_source() argument
262 bio_set_dev(bio, clone->source_dev->bdev); in remap_to_source()
265 static inline void remap_to_dest(struct clone *clone, struct bio *bio) in remap_to_dest() argument
267 bio_set_dev(bio, clone->dest_dev->bdev); in remap_to_dest()
270 static bool bio_triggers_commit(struct clone *clone, struct bio *bio) in bio_triggers_commit() argument
273 dm_clone_changed_this_transaction(clone->cmd); in bio_triggers_commit()
277 static inline sector_t region_to_sector(struct clone *clone, unsigned long region_nr) in region_to_sector() argument
279 return (region_nr << clone->region_shift); in region_to_sector()
283 static inline unsigned long bio_to_region(struct clone *clone, struct bio *bio) in bio_to_region() argument
285 return (bio->bi_iter.bi_sector >> clone->region_shift); in bio_to_region()
289 static void bio_region_range(struct clone *clone, struct bio *bio, in bio_region_range() argument
292 *rs = dm_sector_div_up(bio->bi_iter.bi_sector, clone->region_size); in bio_region_range()
293 *re = bio_end_sector(bio) >> clone->region_shift; in bio_region_range()
297 static inline bool is_overwrite_bio(struct clone *clone, struct bio *bio) in is_overwrite_bio() argument
299 return (bio_data_dir(bio) == WRITE && bio_sectors(bio) == clone->region_size); in is_overwrite_bio()
333 static void issue_bio(struct clone *clone, struct bio *bio) in issue_bio() argument
337 if (!bio_triggers_commit(clone, bio)) { in issue_bio()
346 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) { in issue_bio()
355 spin_lock_irqsave(&clone->lock, flags); in issue_bio()
356 bio_list_add(&clone->deferred_flush_bios, bio); in issue_bio()
357 spin_unlock_irqrestore(&clone->lock, flags); in issue_bio()
359 wake_worker(clone); in issue_bio()
368 static void remap_and_issue(struct clone *clone, struct bio *bio) in remap_and_issue() argument
370 remap_to_dest(clone, bio); in remap_and_issue()
371 issue_bio(clone, bio); in remap_and_issue()
381 static void issue_deferred_bios(struct clone *clone, struct bio_list *bios) in issue_deferred_bios() argument
392 if (bio_triggers_commit(clone, bio)) in issue_deferred_bios()
398 spin_lock_irqsave(&clone->lock, flags); in issue_deferred_bios()
399 bio_list_merge(&clone->deferred_bios, &normal_bios); in issue_deferred_bios()
400 bio_list_merge(&clone->deferred_flush_bios, &flush_bios); in issue_deferred_bios()
401 spin_unlock_irqrestore(&clone->lock, flags); in issue_deferred_bios()
403 wake_worker(clone); in issue_deferred_bios()
406 static void complete_overwrite_bio(struct clone *clone, struct bio *bio) in complete_overwrite_bio() argument
429 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) { in complete_overwrite_bio()
438 spin_lock_irqsave(&clone->lock, flags); in complete_overwrite_bio()
439 bio_list_add(&clone->deferred_flush_completions, bio); in complete_overwrite_bio()
440 spin_unlock_irqrestore(&clone->lock, flags); in complete_overwrite_bio()
442 wake_worker(clone); in complete_overwrite_bio()
451 static void complete_discard_bio(struct clone *clone, struct bio *bio, bool success) in complete_discard_bio() argument
460 if (test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags) && success) { in complete_discard_bio()
461 remap_to_dest(clone, bio); in complete_discard_bio()
462 bio_region_range(clone, bio, &rs, &re); in complete_discard_bio()
463 trim_bio(bio, rs << clone->region_shift, in complete_discard_bio()
464 (re - rs) << clone->region_shift); in complete_discard_bio()
470 static void process_discard_bio(struct clone *clone, struct bio *bio) in process_discard_bio() argument
474 bio_region_range(clone, bio, &rs, &re); in process_discard_bio()
475 BUG_ON(re > clone->nr_regions); in process_discard_bio()
486 if (dm_clone_is_range_hydrated(clone->cmd, rs, re - rs)) { in process_discard_bio()
487 complete_discard_bio(clone, bio, true); in process_discard_bio()
496 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) { in process_discard_bio()
504 spin_lock_irqsave(&clone->lock, flags); in process_discard_bio()
505 bio_list_add(&clone->deferred_discard_bios, bio); in process_discard_bio()
506 spin_unlock_irqrestore(&clone->lock, flags); in process_discard_bio()
508 wake_worker(clone); in process_discard_bio()
517 struct clone *clone; member
557 static int hash_table_init(struct clone *clone) in hash_table_init() argument
564 clone->ht = kvmalloc(sz * sizeof(struct hash_table_bucket), GFP_KERNEL); in hash_table_init()
565 if (!clone->ht) in hash_table_init()
569 bucket = clone->ht + i; in hash_table_init()
578 static void hash_table_exit(struct clone *clone) in hash_table_exit() argument
580 kvfree(clone->ht); in hash_table_exit()
583 static struct hash_table_bucket *get_hash_table_bucket(struct clone *clone, in get_hash_table_bucket() argument
586 return &clone->ht[hash_long(region_nr, HASH_TABLE_BITS)]; in get_hash_table_bucket()
643 static struct dm_clone_region_hydration *alloc_hydration(struct clone *clone) in alloc_hydration() argument
651 hd = mempool_alloc(&clone->hydration_pool, GFP_NOIO); in alloc_hydration()
652 hd->clone = clone; in alloc_hydration()
659 mempool_free(hd, &hd->clone->hydration_pool); in free_hydration()
685 struct clone *clone = hd->clone; in hydration_update_metadata() local
687 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) in hydration_update_metadata()
692 r = dm_clone_set_region_hydrated(clone->cmd, hd->region_nr); in hydration_update_metadata()
694 bucket = get_hash_table_bucket(clone, hd->region_nr); in hydration_update_metadata()
718 struct clone *clone = hd->clone; in hydration_complete() local
724 complete_overwrite_bio(clone, hd->overwrite_bio); in hydration_complete()
726 issue_deferred_bios(clone, &hd->deferred_bios); in hydration_complete()
738 if (atomic_dec_and_test(&clone->hydrations_in_flight)) in hydration_complete()
739 wakeup_hydration_waiters(clone); in hydration_complete()
747 struct clone *clone = hd->clone; in hydration_kcopyd_callback() local
752 DMERR_LIMIT("%s: hydration failed", clone_device_name(clone)); in hydration_kcopyd_callback()
769 if (test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags) && in hydration_kcopyd_callback()
770 !atomic_read(&clone->ios_in_flight)) in hydration_kcopyd_callback()
771 wake_worker(clone); in hydration_kcopyd_callback()
779 struct clone *clone = hd->clone; in hydration_copy() local
781 region_size = clone->region_size; in hydration_copy()
785 total_size = (nr_regions - 1) << clone->region_shift; in hydration_copy()
787 if (region_end == clone->nr_regions - 1) { in hydration_copy()
792 tail_size = clone->ti->len & (region_size - 1); in hydration_copy()
801 from.bdev = clone->source_dev->bdev; in hydration_copy()
802 from.sector = region_to_sector(clone, region_start); in hydration_copy()
805 to.bdev = clone->dest_dev->bdev; in hydration_copy()
810 atomic_add(nr_regions, &clone->hydrations_in_flight); in hydration_copy()
811 dm_kcopyd_copy(clone->kcopyd_client, &from, 1, &to, 0, in hydration_copy()
838 atomic_inc(&hd->clone->hydrations_in_flight); in hydration_overwrite()
852 static void hydrate_bio_region(struct clone *clone, struct bio *bio) in hydrate_bio_region() argument
859 region_nr = bio_to_region(clone, bio); in hydrate_bio_region()
860 bucket = get_hash_table_bucket(clone, region_nr); in hydrate_bio_region()
872 if (dm_clone_is_region_hydrated(clone->cmd, region_nr)) { in hydrate_bio_region()
875 issue_bio(clone, bio); in hydrate_bio_region()
885 hd = alloc_hydration(clone); in hydrate_bio_region()
891 if (dm_clone_is_region_hydrated(clone->cmd, region_nr)) { in hydrate_bio_region()
894 issue_bio(clone, bio); in hydrate_bio_region()
912 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) { in hydrate_bio_region()
927 if (is_overwrite_bio(clone, bio)) { in hydrate_bio_region()
960 struct clone *clone = hd->clone; in __batch_hydration() local
961 unsigned int max_batch_size = READ_ONCE(clone->hydration_batch_size); in __batch_hydration()
995 static unsigned long __start_next_hydration(struct clone *clone, in __start_next_hydration() argument
1002 unsigned long nr_regions = clone->nr_regions; in __start_next_hydration()
1004 hd = alloc_hydration(clone); in __start_next_hydration()
1008 offset = dm_clone_find_next_unhydrated_region(clone->cmd, offset); in __start_next_hydration()
1012 bucket = get_hash_table_bucket(clone, offset); in __start_next_hydration()
1015 if (!dm_clone_is_region_hydrated(clone->cmd, offset) && in __start_next_hydration()
1041 static void do_hydration(struct clone *clone) in do_hydration() argument
1044 unsigned long offset, nr_regions = clone->nr_regions; in do_hydration()
1051 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) in do_hydration()
1054 if (dm_clone_is_hydration_done(clone->cmd)) in do_hydration()
1060 atomic_inc(&clone->hydrations_in_flight); in do_hydration()
1072 offset = clone->hydration_offset; in do_hydration()
1073 while (likely(!test_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags)) && in do_hydration()
1074 !atomic_read(&clone->ios_in_flight) && in do_hydration()
1075 test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags) && in do_hydration()
1077 current_volume = atomic_read(&clone->hydrations_in_flight); in do_hydration()
1080 if (current_volume > READ_ONCE(clone->hydration_threshold)) in do_hydration()
1083 offset = __start_next_hydration(clone, offset, &batch); in do_hydration()
1092 clone->hydration_offset = offset; in do_hydration()
1094 if (atomic_dec_and_test(&clone->hydrations_in_flight)) in do_hydration()
1095 wakeup_hydration_waiters(clone); in do_hydration()
1100 static bool need_commit_due_to_time(struct clone *clone) in need_commit_due_to_time() argument
1102 return !time_in_range(jiffies, clone->last_commit_jiffies, in need_commit_due_to_time()
1103 clone->last_commit_jiffies + COMMIT_PERIOD); in need_commit_due_to_time()
1109 static int commit_metadata(struct clone *clone) in commit_metadata() argument
1113 mutex_lock(&clone->commit_lock); in commit_metadata()
1115 if (!dm_clone_changed_this_transaction(clone->cmd)) in commit_metadata()
1118 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) { in commit_metadata()
1123 r = dm_clone_metadata_commit(clone->cmd); in commit_metadata()
1126 __metadata_operation_failed(clone, "dm_clone_metadata_commit", r); in commit_metadata()
1130 if (dm_clone_is_hydration_done(clone->cmd)) in commit_metadata()
1131 dm_table_event(clone->ti->table); in commit_metadata()
1133 mutex_unlock(&clone->commit_lock); in commit_metadata()
1138 static void process_deferred_discards(struct clone *clone) in process_deferred_discards() argument
1146 spin_lock_irqsave(&clone->lock, flags); in process_deferred_discards()
1147 bio_list_merge(&discards, &clone->deferred_discard_bios); in process_deferred_discards()
1148 bio_list_init(&clone->deferred_discard_bios); in process_deferred_discards()
1149 spin_unlock_irqrestore(&clone->lock, flags); in process_deferred_discards()
1154 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) in process_deferred_discards()
1159 bio_region_range(clone, bio, &rs, &re); in process_deferred_discards()
1165 r = dm_clone_cond_set_range(clone->cmd, rs, re - rs); in process_deferred_discards()
1173 complete_discard_bio(clone, bio, r == 0); in process_deferred_discards()
1177 static void process_deferred_bios(struct clone *clone) in process_deferred_bios() argument
1182 spin_lock_irqsave(&clone->lock, flags); in process_deferred_bios()
1183 bio_list_merge(&bios, &clone->deferred_bios); in process_deferred_bios()
1184 bio_list_init(&clone->deferred_bios); in process_deferred_bios()
1185 spin_unlock_irqrestore(&clone->lock, flags); in process_deferred_bios()
1193 static void process_deferred_flush_bios(struct clone *clone) in process_deferred_flush_bios() argument
1204 spin_lock_irqsave(&clone->lock, flags); in process_deferred_flush_bios()
1205 bio_list_merge(&bios, &clone->deferred_flush_bios); in process_deferred_flush_bios()
1206 bio_list_init(&clone->deferred_flush_bios); in process_deferred_flush_bios()
1208 bio_list_merge(&bio_completions, &clone->deferred_flush_completions); in process_deferred_flush_bios()
1209 bio_list_init(&clone->deferred_flush_completions); in process_deferred_flush_bios()
1210 spin_unlock_irqrestore(&clone->lock, flags); in process_deferred_flush_bios()
1213 !(dm_clone_changed_this_transaction(clone->cmd) && need_commit_due_to_time(clone))) in process_deferred_flush_bios()
1216 if (commit_metadata(clone)) { in process_deferred_flush_bios()
1225 clone->last_commit_jiffies = jiffies; in process_deferred_flush_bios()
1236 struct clone *clone = container_of(work, typeof(*clone), worker); in do_worker() local
1238 process_deferred_bios(clone); in do_worker()
1239 process_deferred_discards(clone); in do_worker()
1250 process_deferred_flush_bios(clone); in do_worker()
1253 do_hydration(clone); in do_worker()
1263 struct clone *clone = container_of(to_delayed_work(work), struct clone, waker); in do_waker() local
1265 wake_worker(clone); in do_waker()
1266 queue_delayed_work(clone->wq, &clone->waker, COMMIT_PERIOD); in do_waker()
1276 struct clone *clone = ti->private; in clone_map() local
1279 atomic_inc(&clone->ios_in_flight); in clone_map()
1281 if (unlikely(get_clone_mode(clone) == CM_FAIL)) in clone_map()
1292 remap_and_issue(clone, bio); in clone_map()
1304 process_discard_bio(clone, bio); in clone_map()
1318 region_nr = bio_to_region(clone, bio); in clone_map()
1319 if (dm_clone_is_region_hydrated(clone->cmd, region_nr)) { in clone_map()
1320 remap_and_issue(clone, bio); in clone_map()
1323 remap_to_source(clone, bio); in clone_map()
1327 remap_to_dest(clone, bio); in clone_map()
1328 hydrate_bio_region(clone, bio); in clone_map()
1335 struct clone *clone = ti->private; in clone_endio() local
1337 atomic_dec(&clone->ios_in_flight); in clone_endio()
1342 static void emit_flags(struct clone *clone, char *result, unsigned int maxlen, in emit_flags() argument
1348 count = !test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags); in emit_flags()
1349 count += !test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags); in emit_flags()
1353 if (!test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags)) in emit_flags()
1356 if (!test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags)) in emit_flags()
1362 static void emit_core_args(struct clone *clone, char *result, in emit_core_args() argument
1369 READ_ONCE(clone->hydration_threshold), in emit_core_args()
1370 READ_ONCE(clone->hydration_batch_size)); in emit_core_args()
1392 struct clone *clone = ti->private; in clone_status() local
1396 if (get_clone_mode(clone) == CM_FAIL) { in clone_status()
1403 (void) commit_metadata(clone); in clone_status()
1405 r = dm_clone_get_free_metadata_block_count(clone->cmd, &nr_free_metadata_blocks); in clone_status()
1409 clone_device_name(clone), r); in clone_status()
1413 r = dm_clone_get_metadata_dev_size(clone->cmd, &nr_metadata_blocks); in clone_status()
1417 clone_device_name(clone), r); in clone_status()
1425 (unsigned long long)clone->region_size, in clone_status()
1426 dm_clone_nr_of_hydrated_regions(clone->cmd), in clone_status()
1427 clone->nr_regions, in clone_status()
1428 atomic_read(&clone->hydrations_in_flight)); in clone_status()
1430 emit_flags(clone, result, maxlen, &sz); in clone_status()
1431 emit_core_args(clone, result, maxlen, &sz); in clone_status()
1433 switch (get_clone_mode(clone)) { in clone_status()
1447 format_dev_t(buf, clone->metadata_dev->bdev->bd_dev); in clone_status()
1450 format_dev_t(buf, clone->dest_dev->bdev->bd_dev); in clone_status()
1453 format_dev_t(buf, clone->source_dev->bdev->bd_dev); in clone_status()
1456 for (i = 0; i < clone->nr_ctr_args; i++) in clone_status()
1457 DMEMIT(" %s", clone->ctr_args[i]); in clone_status()
1469 struct clone *clone = container_of(cb, struct clone, callbacks); in clone_is_congested() local
1471 source_q = bdev_get_queue(clone->source_dev->bdev); in clone_is_congested()
1472 dest_q = bdev_get_queue(clone->dest_dev->bdev); in clone_is_congested()
1504 static int parse_feature_args(struct dm_arg_set *as, struct clone *clone) in parse_feature_args() argument
1509 struct dm_target *ti = clone->ti; in parse_feature_args()
1530 __clear_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags); in parse_feature_args()
1532 __clear_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags); in parse_feature_args()
1542 static int parse_core_args(struct dm_arg_set *as, struct clone *clone) in parse_core_args() argument
1548 struct dm_target *ti = clone->ti; in parse_core_args()
1557 clone->hydration_batch_size = DEFAULT_HYDRATION_BATCH_SIZE; in parse_core_args()
1558 clone->hydration_threshold = DEFAULT_HYDRATION_THRESHOLD; in parse_core_args()
1582 clone->hydration_threshold = value; in parse_core_args()
1588 clone->hydration_batch_size = value; in parse_core_args()
1598 static int parse_region_size(struct clone *clone, struct dm_arg_set *as, char **error) in parse_region_size() argument
1619 if (region_size % (bdev_logical_block_size(clone->source_dev->bdev) >> 9) || in parse_region_size()
1620 region_size % (bdev_logical_block_size(clone->dest_dev->bdev) >> 9)) { in parse_region_size()
1625 clone->region_size = region_size; in parse_region_size()
1644 static int parse_metadata_dev(struct clone *clone, struct dm_arg_set *as, char **error) in parse_metadata_dev() argument
1650 r = dm_get_device(clone->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE, in parse_metadata_dev()
1651 &clone->metadata_dev); in parse_metadata_dev()
1657 metadata_dev_size = get_dev_size(clone->metadata_dev); in parse_metadata_dev()
1660 bdevname(clone->metadata_dev->bdev, b), DM_CLONE_METADATA_MAX_SECTORS); in parse_metadata_dev()
1665 static int parse_dest_dev(struct clone *clone, struct dm_arg_set *as, char **error) in parse_dest_dev() argument
1670 r = dm_get_device(clone->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE, in parse_dest_dev()
1671 &clone->dest_dev); in parse_dest_dev()
1677 dest_dev_size = get_dev_size(clone->dest_dev); in parse_dest_dev()
1678 if (dest_dev_size < clone->ti->len) { in parse_dest_dev()
1679 dm_put_device(clone->ti, clone->dest_dev); in parse_dest_dev()
1687 static int parse_source_dev(struct clone *clone, struct dm_arg_set *as, char **error) in parse_source_dev() argument
1692 r = dm_get_device(clone->ti, dm_shift_arg(as), FMODE_READ, in parse_source_dev()
1693 &clone->source_dev); in parse_source_dev()
1699 source_dev_size = get_dev_size(clone->source_dev); in parse_source_dev()
1700 if (source_dev_size < clone->ti->len) { in parse_source_dev()
1701 dm_put_device(clone->ti, clone->source_dev); in parse_source_dev()
1709 static int copy_ctr_args(struct clone *clone, int argc, const char **argv, char **error) in copy_ctr_args() argument
1729 clone->nr_ctr_args = argc; in copy_ctr_args()
1730 clone->ctr_args = copy; in copy_ctr_args()
1741 struct clone *clone; in clone_ctr() local
1752 clone = kzalloc(sizeof(*clone), GFP_KERNEL); in clone_ctr()
1753 if (!clone) { in clone_ctr()
1758 clone->ti = ti; in clone_ctr()
1761 __set_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags); in clone_ctr()
1762 __set_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags); in clone_ctr()
1763 __set_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags); in clone_ctr()
1765 r = parse_metadata_dev(clone, &as, &ti->error); in clone_ctr()
1769 r = parse_dest_dev(clone, &as, &ti->error); in clone_ctr()
1773 r = parse_source_dev(clone, &as, &ti->error); in clone_ctr()
1777 r = parse_region_size(clone, &as, &ti->error); in clone_ctr()
1781 clone->region_shift = __ffs(clone->region_size); in clone_ctr()
1782 clone->nr_regions = dm_sector_div_up(ti->len, clone->region_size); in clone_ctr()
1784 r = validate_nr_regions(clone->nr_regions, &ti->error); in clone_ctr()
1788 r = dm_set_target_max_io_len(ti, clone->region_size); in clone_ctr()
1794 r = parse_feature_args(&as, clone); in clone_ctr()
1798 r = parse_core_args(&as, clone); in clone_ctr()
1803 clone->cmd = dm_clone_metadata_open(clone->metadata_dev->bdev, ti->len, in clone_ctr()
1804 clone->region_size); in clone_ctr()
1805 if (IS_ERR(clone->cmd)) { in clone_ctr()
1807 r = PTR_ERR(clone->cmd); in clone_ctr()
1811 __set_clone_mode(clone, CM_WRITE); in clone_ctr()
1813 if (get_clone_mode(clone) != CM_WRITE) { in clone_ctr()
1819 clone->last_commit_jiffies = jiffies; in clone_ctr()
1822 r = hash_table_init(clone); in clone_ctr()
1828 atomic_set(&clone->ios_in_flight, 0); in clone_ctr()
1829 init_waitqueue_head(&clone->hydration_stopped); in clone_ctr()
1830 spin_lock_init(&clone->lock); in clone_ctr()
1831 bio_list_init(&clone->deferred_bios); in clone_ctr()
1832 bio_list_init(&clone->deferred_discard_bios); in clone_ctr()
1833 bio_list_init(&clone->deferred_flush_bios); in clone_ctr()
1834 bio_list_init(&clone->deferred_flush_completions); in clone_ctr()
1835 clone->hydration_offset = 0; in clone_ctr()
1836 atomic_set(&clone->hydrations_in_flight, 0); in clone_ctr()
1838 clone->wq = alloc_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM, 0); in clone_ctr()
1839 if (!clone->wq) { in clone_ctr()
1845 INIT_WORK(&clone->worker, do_worker); in clone_ctr()
1846 INIT_DELAYED_WORK(&clone->waker, do_waker); in clone_ctr()
1848 clone->kcopyd_client = dm_kcopyd_client_create(&dm_kcopyd_throttle); in clone_ctr()
1849 if (IS_ERR(clone->kcopyd_client)) { in clone_ctr()
1850 r = PTR_ERR(clone->kcopyd_client); in clone_ctr()
1854 r = mempool_init_slab_pool(&clone->hydration_pool, MIN_HYDRATIONS, in clone_ctr()
1862 r = copy_ctr_args(clone, argc - 3, (const char **)argv + 3, &ti->error); in clone_ctr()
1866 mutex_init(&clone->commit_lock); in clone_ctr()
1867 clone->callbacks.congested_fn = clone_is_congested; in clone_ctr()
1868 dm_table_add_target_callbacks(ti->table, &clone->callbacks); in clone_ctr()
1878 ti->private = clone; in clone_ctr()
1883 mempool_exit(&clone->hydration_pool); in clone_ctr()
1885 dm_kcopyd_client_destroy(clone->kcopyd_client); in clone_ctr()
1887 destroy_workqueue(clone->wq); in clone_ctr()
1889 hash_table_exit(clone); in clone_ctr()
1891 dm_clone_metadata_close(clone->cmd); in clone_ctr()
1893 dm_put_device(ti, clone->source_dev); in clone_ctr()
1895 dm_put_device(ti, clone->dest_dev); in clone_ctr()
1897 dm_put_device(ti, clone->metadata_dev); in clone_ctr()
1899 kfree(clone); in clone_ctr()
1907 struct clone *clone = ti->private; in clone_dtr() local
1909 mutex_destroy(&clone->commit_lock); in clone_dtr()
1911 for (i = 0; i < clone->nr_ctr_args; i++) in clone_dtr()
1912 kfree(clone->ctr_args[i]); in clone_dtr()
1913 kfree(clone->ctr_args); in clone_dtr()
1915 mempool_exit(&clone->hydration_pool); in clone_dtr()
1916 dm_kcopyd_client_destroy(clone->kcopyd_client); in clone_dtr()
1917 destroy_workqueue(clone->wq); in clone_dtr()
1918 hash_table_exit(clone); in clone_dtr()
1919 dm_clone_metadata_close(clone->cmd); in clone_dtr()
1920 dm_put_device(ti, clone->source_dev); in clone_dtr()
1921 dm_put_device(ti, clone->dest_dev); in clone_dtr()
1922 dm_put_device(ti, clone->metadata_dev); in clone_dtr()
1924 kfree(clone); in clone_dtr()
1931 struct clone *clone = ti->private; in clone_postsuspend() local
1948 cancel_delayed_work_sync(&clone->waker); in clone_postsuspend()
1950 set_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags); in clone_postsuspend()
1961 wait_event(clone->hydration_stopped, !atomic_read(&clone->hydrations_in_flight)); in clone_postsuspend()
1962 flush_workqueue(clone->wq); in clone_postsuspend()
1964 (void) commit_metadata(clone); in clone_postsuspend()
1969 struct clone *clone = ti->private; in clone_resume() local
1971 clear_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags); in clone_resume()
1972 do_waker(&clone->waker.work); in clone_resume()
1986 static void disable_passdown_if_not_supported(struct clone *clone) in disable_passdown_if_not_supported() argument
1988 struct block_device *dest_dev = clone->dest_dev->bdev; in disable_passdown_if_not_supported()
1993 if (!test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags)) in disable_passdown_if_not_supported()
1998 else if (dest_limits->max_discard_sectors < clone->region_size) in disable_passdown_if_not_supported()
2004 clear_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags); in disable_passdown_if_not_supported()
2008 static void set_discard_limits(struct clone *clone, struct queue_limits *limits) in set_discard_limits() argument
2010 struct block_device *dest_bdev = clone->dest_dev->bdev; in set_discard_limits()
2013 if (!test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags)) { in set_discard_limits()
2015 limits->discard_granularity = clone->region_size << SECTOR_SHIFT; in set_discard_limits()
2016 limits->max_discard_sectors = round_down(UINT_MAX >> SECTOR_SHIFT, clone->region_size); in set_discard_limits()
2035 struct clone *clone = ti->private; in clone_io_hints() local
2042 if (io_opt_sectors < clone->region_size || in clone_io_hints()
2043 do_div(io_opt_sectors, clone->region_size)) { in clone_io_hints()
2044 blk_limits_io_min(limits, clone->region_size << SECTOR_SHIFT); in clone_io_hints()
2045 blk_limits_io_opt(limits, clone->region_size << SECTOR_SHIFT); in clone_io_hints()
2048 disable_passdown_if_not_supported(clone); in clone_io_hints()
2049 set_discard_limits(clone, limits); in clone_io_hints()
2056 struct clone *clone = ti->private; in clone_iterate_devices() local
2057 struct dm_dev *dest_dev = clone->dest_dev; in clone_iterate_devices()
2058 struct dm_dev *source_dev = clone->source_dev; in clone_iterate_devices()
2069 static void set_hydration_threshold(struct clone *clone, unsigned int nr_regions) in set_hydration_threshold() argument
2071 WRITE_ONCE(clone->hydration_threshold, nr_regions); in set_hydration_threshold()
2078 wake_worker(clone); in set_hydration_threshold()
2081 static void set_hydration_batch_size(struct clone *clone, unsigned int nr_regions) in set_hydration_batch_size() argument
2083 WRITE_ONCE(clone->hydration_batch_size, nr_regions); in set_hydration_batch_size()
2086 static void enable_hydration(struct clone *clone) in enable_hydration() argument
2088 if (!test_and_set_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags)) in enable_hydration()
2089 wake_worker(clone); in enable_hydration()
2092 static void disable_hydration(struct clone *clone) in disable_hydration() argument
2094 clear_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags); in disable_hydration()
2100 struct clone *clone = ti->private; in clone_message() local
2107 enable_hydration(clone); in clone_message()
2112 disable_hydration(clone); in clone_message()
2123 set_hydration_threshold(clone, value); in clone_message()
2132 set_hydration_batch_size(clone, value); in clone_message()
2137 DMERR("%s: Unsupported message `%s'", clone_device_name(clone), argv[0]); in clone_message()