Lines Matching refs:rh
106 struct dm_region_hash *rh; /* FIXME: can we get rid of this ? */ member
120 static region_t dm_rh_sector_to_region(struct dm_region_hash *rh, sector_t sector) in dm_rh_sector_to_region() argument
122 return sector >> rh->region_shift; in dm_rh_sector_to_region()
125 sector_t dm_rh_region_to_sector(struct dm_region_hash *rh, region_t region) in dm_rh_region_to_sector() argument
127 return region << rh->region_shift; in dm_rh_region_to_sector()
131 region_t dm_rh_bio_to_region(struct dm_region_hash *rh, struct bio *bio) in dm_rh_bio_to_region() argument
133 return dm_rh_sector_to_region(rh, bio->bi_iter.bi_sector - in dm_rh_bio_to_region()
134 rh->target_begin); in dm_rh_bio_to_region()
140 return reg->rh->context; in dm_rh_region_context()
150 sector_t dm_rh_get_region_size(struct dm_region_hash *rh) in dm_rh_get_region_size() argument
152 return rh->region_size; in dm_rh_get_region_size()
173 struct dm_region_hash *rh; in dm_region_hash_create() local
187 rh = kzalloc(sizeof(*rh), GFP_KERNEL); in dm_region_hash_create()
188 if (!rh) { in dm_region_hash_create()
193 rh->context = context; in dm_region_hash_create()
194 rh->dispatch_bios = dispatch_bios; in dm_region_hash_create()
195 rh->wakeup_workers = wakeup_workers; in dm_region_hash_create()
196 rh->wakeup_all_recovery_waiters = wakeup_all_recovery_waiters; in dm_region_hash_create()
197 rh->target_begin = target_begin; in dm_region_hash_create()
198 rh->max_recovery = max_recovery; in dm_region_hash_create()
199 rh->log = log; in dm_region_hash_create()
200 rh->region_size = region_size; in dm_region_hash_create()
201 rh->region_shift = __ffs(region_size); in dm_region_hash_create()
202 rwlock_init(&rh->hash_lock); in dm_region_hash_create()
203 rh->mask = nr_buckets - 1; in dm_region_hash_create()
204 rh->nr_buckets = nr_buckets; in dm_region_hash_create()
206 rh->shift = RH_HASH_SHIFT; in dm_region_hash_create()
207 rh->prime = RH_HASH_MULT; in dm_region_hash_create()
209 rh->buckets = vmalloc(array_size(nr_buckets, sizeof(*rh->buckets))); in dm_region_hash_create()
210 if (!rh->buckets) { in dm_region_hash_create()
212 kfree(rh); in dm_region_hash_create()
217 INIT_LIST_HEAD(rh->buckets + i); in dm_region_hash_create()
219 spin_lock_init(&rh->region_lock); in dm_region_hash_create()
220 sema_init(&rh->recovery_count, 0); in dm_region_hash_create()
221 atomic_set(&rh->recovery_in_flight, 0); in dm_region_hash_create()
222 INIT_LIST_HEAD(&rh->clean_regions); in dm_region_hash_create()
223 INIT_LIST_HEAD(&rh->quiesced_regions); in dm_region_hash_create()
224 INIT_LIST_HEAD(&rh->recovered_regions); in dm_region_hash_create()
225 INIT_LIST_HEAD(&rh->failed_recovered_regions); in dm_region_hash_create()
226 rh->flush_failure = 0; in dm_region_hash_create()
228 ret = mempool_init_kmalloc_pool(&rh->region_pool, MIN_REGIONS, in dm_region_hash_create()
231 vfree(rh->buckets); in dm_region_hash_create()
232 kfree(rh); in dm_region_hash_create()
233 rh = ERR_PTR(-ENOMEM); in dm_region_hash_create()
236 return rh; in dm_region_hash_create()
240 void dm_region_hash_destroy(struct dm_region_hash *rh) in dm_region_hash_destroy() argument
245 BUG_ON(!list_empty(&rh->quiesced_regions)); in dm_region_hash_destroy()
246 for (h = 0; h < rh->nr_buckets; h++) { in dm_region_hash_destroy()
247 list_for_each_entry_safe(reg, nreg, rh->buckets + h, in dm_region_hash_destroy()
250 mempool_free(reg, &rh->region_pool); in dm_region_hash_destroy()
254 if (rh->log) in dm_region_hash_destroy()
255 dm_dirty_log_destroy(rh->log); in dm_region_hash_destroy()
257 mempool_exit(&rh->region_pool); in dm_region_hash_destroy()
258 vfree(rh->buckets); in dm_region_hash_destroy()
259 kfree(rh); in dm_region_hash_destroy()
263 struct dm_dirty_log *dm_rh_dirty_log(struct dm_region_hash *rh) in dm_rh_dirty_log() argument
265 return rh->log; in dm_rh_dirty_log()
269 static unsigned int rh_hash(struct dm_region_hash *rh, region_t region) in rh_hash() argument
271 return (unsigned int) ((region * rh->prime) >> rh->shift) & rh->mask; in rh_hash()
274 static struct dm_region *__rh_lookup(struct dm_region_hash *rh, region_t region) in __rh_lookup() argument
277 struct list_head *bucket = rh->buckets + rh_hash(rh, region); in __rh_lookup()
286 static void __rh_insert(struct dm_region_hash *rh, struct dm_region *reg) in __rh_insert() argument
288 list_add(®->hash_list, rh->buckets + rh_hash(rh, reg->key)); in __rh_insert()
291 static struct dm_region *__rh_alloc(struct dm_region_hash *rh, region_t region) in __rh_alloc() argument
295 nreg = mempool_alloc(&rh->region_pool, GFP_ATOMIC); in __rh_alloc()
299 nreg->state = rh->log->type->in_sync(rh->log, region, 1) ? in __rh_alloc()
301 nreg->rh = rh; in __rh_alloc()
307 write_lock_irq(&rh->hash_lock); in __rh_alloc()
308 reg = __rh_lookup(rh, region); in __rh_alloc()
311 mempool_free(nreg, &rh->region_pool); in __rh_alloc()
313 __rh_insert(rh, nreg); in __rh_alloc()
315 spin_lock(&rh->region_lock); in __rh_alloc()
316 list_add(&nreg->list, &rh->clean_regions); in __rh_alloc()
317 spin_unlock(&rh->region_lock); in __rh_alloc()
322 write_unlock_irq(&rh->hash_lock); in __rh_alloc()
327 static struct dm_region *__rh_find(struct dm_region_hash *rh, region_t region) in __rh_find() argument
331 reg = __rh_lookup(rh, region); in __rh_find()
333 read_unlock(&rh->hash_lock); in __rh_find()
334 reg = __rh_alloc(rh, region); in __rh_find()
335 read_lock(&rh->hash_lock); in __rh_find()
341 int dm_rh_get_state(struct dm_region_hash *rh, region_t region, int may_block) in dm_rh_get_state() argument
346 read_lock(&rh->hash_lock); in dm_rh_get_state()
347 reg = __rh_lookup(rh, region); in dm_rh_get_state()
348 read_unlock(&rh->hash_lock); in dm_rh_get_state()
357 r = rh->log->type->in_sync(rh->log, region, may_block); in dm_rh_get_state()
369 struct dm_region_hash *rh = reg->rh; in complete_resync_work() local
371 rh->log->type->set_region_sync(rh->log, reg->key, success); in complete_resync_work()
382 rh->dispatch_bios(rh->context, ®->delayed_bios); in complete_resync_work()
383 if (atomic_dec_and_test(&rh->recovery_in_flight)) in complete_resync_work()
384 rh->wakeup_all_recovery_waiters(rh->context); in complete_resync_work()
385 up(&rh->recovery_count); in complete_resync_work()
398 void dm_rh_mark_nosync(struct dm_region_hash *rh, struct bio *bio) in dm_rh_mark_nosync() argument
401 struct dm_dirty_log *log = rh->log; in dm_rh_mark_nosync()
403 region_t region = dm_rh_bio_to_region(rh, bio); in dm_rh_mark_nosync()
407 rh->flush_failure = 1; in dm_rh_mark_nosync()
417 read_lock(&rh->hash_lock); in dm_rh_mark_nosync()
418 reg = __rh_find(rh, region); in dm_rh_mark_nosync()
419 read_unlock(&rh->hash_lock); in dm_rh_mark_nosync()
425 spin_lock_irqsave(&rh->region_lock, flags); in dm_rh_mark_nosync()
436 spin_unlock_irqrestore(&rh->region_lock, flags); in dm_rh_mark_nosync()
443 void dm_rh_update_states(struct dm_region_hash *rh, int errors_handled) in dm_rh_update_states() argument
454 write_lock_irq(&rh->hash_lock); in dm_rh_update_states()
455 spin_lock(&rh->region_lock); in dm_rh_update_states()
456 if (!list_empty(&rh->clean_regions)) { in dm_rh_update_states()
457 list_splice_init(&rh->clean_regions, &clean); in dm_rh_update_states()
463 if (!list_empty(&rh->recovered_regions)) { in dm_rh_update_states()
464 list_splice_init(&rh->recovered_regions, &recovered); in dm_rh_update_states()
470 if (!list_empty(&rh->failed_recovered_regions)) { in dm_rh_update_states()
471 list_splice_init(&rh->failed_recovered_regions, in dm_rh_update_states()
478 spin_unlock(&rh->region_lock); in dm_rh_update_states()
479 write_unlock_irq(&rh->hash_lock); in dm_rh_update_states()
487 rh->log->type->clear_region(rh->log, reg->key); in dm_rh_update_states()
489 mempool_free(reg, &rh->region_pool); in dm_rh_update_states()
494 mempool_free(reg, &rh->region_pool); in dm_rh_update_states()
498 rh->log->type->clear_region(rh->log, reg->key); in dm_rh_update_states()
499 mempool_free(reg, &rh->region_pool); in dm_rh_update_states()
502 rh->log->type->flush(rh->log); in dm_rh_update_states()
506 static void rh_inc(struct dm_region_hash *rh, region_t region) in rh_inc() argument
510 read_lock(&rh->hash_lock); in rh_inc()
511 reg = __rh_find(rh, region); in rh_inc()
513 spin_lock_irq(&rh->region_lock); in rh_inc()
519 spin_unlock_irq(&rh->region_lock); in rh_inc()
521 rh->log->type->mark_region(rh->log, reg->key); in rh_inc()
523 spin_unlock_irq(&rh->region_lock); in rh_inc()
526 read_unlock(&rh->hash_lock); in rh_inc()
529 void dm_rh_inc_pending(struct dm_region_hash *rh, struct bio_list *bios) in dm_rh_inc_pending() argument
536 rh_inc(rh, dm_rh_bio_to_region(rh, bio)); in dm_rh_inc_pending()
541 void dm_rh_dec(struct dm_region_hash *rh, region_t region) in dm_rh_dec() argument
547 read_lock(&rh->hash_lock); in dm_rh_dec()
548 reg = __rh_lookup(rh, region); in dm_rh_dec()
549 read_unlock(&rh->hash_lock); in dm_rh_dec()
551 spin_lock_irqsave(&rh->region_lock, flags); in dm_rh_dec()
565 if (unlikely(rh->flush_failure)) { in dm_rh_dec()
573 list_add_tail(®->list, &rh->quiesced_regions); in dm_rh_dec()
576 list_add(®->list, &rh->clean_regions); in dm_rh_dec()
580 spin_unlock_irqrestore(&rh->region_lock, flags); in dm_rh_dec()
583 rh->wakeup_workers(rh->context); in dm_rh_dec()
590 static int __rh_recovery_prepare(struct dm_region_hash *rh) in __rh_recovery_prepare() argument
599 r = rh->log->type->get_resync_work(rh->log, ®ion); in __rh_recovery_prepare()
607 read_lock(&rh->hash_lock); in __rh_recovery_prepare()
608 reg = __rh_find(rh, region); in __rh_recovery_prepare()
609 read_unlock(&rh->hash_lock); in __rh_recovery_prepare()
611 spin_lock_irq(&rh->region_lock); in __rh_recovery_prepare()
618 list_move(®->list, &rh->quiesced_regions); in __rh_recovery_prepare()
620 spin_unlock_irq(&rh->region_lock); in __rh_recovery_prepare()
625 void dm_rh_recovery_prepare(struct dm_region_hash *rh) in dm_rh_recovery_prepare() argument
628 atomic_inc(&rh->recovery_in_flight); in dm_rh_recovery_prepare()
630 while (!down_trylock(&rh->recovery_count)) { in dm_rh_recovery_prepare()
631 atomic_inc(&rh->recovery_in_flight); in dm_rh_recovery_prepare()
632 if (__rh_recovery_prepare(rh) <= 0) { in dm_rh_recovery_prepare()
633 atomic_dec(&rh->recovery_in_flight); in dm_rh_recovery_prepare()
634 up(&rh->recovery_count); in dm_rh_recovery_prepare()
640 if (atomic_dec_and_test(&rh->recovery_in_flight)) in dm_rh_recovery_prepare()
641 rh->wakeup_all_recovery_waiters(rh->context); in dm_rh_recovery_prepare()
648 struct dm_region *dm_rh_recovery_start(struct dm_region_hash *rh) in dm_rh_recovery_start() argument
652 spin_lock_irq(&rh->region_lock); in dm_rh_recovery_start()
653 if (!list_empty(&rh->quiesced_regions)) { in dm_rh_recovery_start()
654 reg = list_entry(rh->quiesced_regions.next, in dm_rh_recovery_start()
658 spin_unlock_irq(&rh->region_lock); in dm_rh_recovery_start()
666 struct dm_region_hash *rh = reg->rh; in dm_rh_recovery_end() local
668 spin_lock_irq(&rh->region_lock); in dm_rh_recovery_end()
670 list_add(®->list, ®->rh->recovered_regions); in dm_rh_recovery_end()
672 list_add(®->list, ®->rh->failed_recovered_regions); in dm_rh_recovery_end()
674 spin_unlock_irq(&rh->region_lock); in dm_rh_recovery_end()
676 rh->wakeup_workers(rh->context); in dm_rh_recovery_end()
681 int dm_rh_recovery_in_flight(struct dm_region_hash *rh) in dm_rh_recovery_in_flight() argument
683 return atomic_read(&rh->recovery_in_flight); in dm_rh_recovery_in_flight()
687 int dm_rh_flush(struct dm_region_hash *rh) in dm_rh_flush() argument
689 return rh->log->type->flush(rh->log); in dm_rh_flush()
693 void dm_rh_delay(struct dm_region_hash *rh, struct bio *bio) in dm_rh_delay() argument
697 read_lock(&rh->hash_lock); in dm_rh_delay()
698 reg = __rh_find(rh, dm_rh_bio_to_region(rh, bio)); in dm_rh_delay()
700 read_unlock(&rh->hash_lock); in dm_rh_delay()
704 void dm_rh_stop_recovery(struct dm_region_hash *rh) in dm_rh_stop_recovery() argument
709 for (i = 0; i < rh->max_recovery; i++) in dm_rh_stop_recovery()
710 down(&rh->recovery_count); in dm_rh_stop_recovery()
714 void dm_rh_start_recovery(struct dm_region_hash *rh) in dm_rh_start_recovery() argument
718 for (i = 0; i < rh->max_recovery; i++) in dm_rh_start_recovery()
719 up(&rh->recovery_count); in dm_rh_start_recovery()
721 rh->wakeup_workers(rh->context); in dm_rh_start_recovery()