137 static inline sector_t bio_to_dev_sectors(struct log_writes_c *lc,  in bio_to_dev_sectors()  argument
140 	return sectors >> (lc->sectorshift - SECTOR_SHIFT);  in bio_to_dev_sectors()
143 static inline sector_t dev_to_bio_sectors(struct log_writes_c *lc,  in dev_to_bio_sectors()  argument
146 	return sectors << (lc->sectorshift - SECTOR_SHIFT);  in dev_to_bio_sectors()
149 static void put_pending_block(struct log_writes_c *lc)  in put_pending_block()  argument
151 	if (atomic_dec_and_test(&lc->pending_blocks)) {  in put_pending_block()
153 		if (waitqueue_active(&lc->wait))  in put_pending_block()
154 			wake_up(&lc->wait);  in put_pending_block()
158 static void put_io_block(struct log_writes_c *lc)  in put_io_block()  argument
160 	if (atomic_dec_and_test(&lc->io_blocks)) {  in put_io_block()
162 		if (waitqueue_active(&lc->wait))  in put_io_block()
163 			wake_up(&lc->wait);  in put_io_block()
169 	struct log_writes_c *lc = bio->bi_private;  in log_end_io()  local
175 		spin_lock_irqsave(&lc->blocks_lock, flags);  in log_end_io()
176 		lc->logging_enabled = false;  in log_end_io()
177 		spin_unlock_irqrestore(&lc->blocks_lock, flags);  in log_end_io()
181 	put_io_block(lc);  in log_end_io()
187 	struct log_writes_c *lc = bio->bi_private;  in log_end_super()  local
189 	complete(&lc->super_done);  in log_end_super()
197 static void free_pending_block(struct log_writes_c *lc,  in free_pending_block()  argument
208 	put_pending_block(lc);  in free_pending_block()
211 static int write_metadata(struct log_writes_c *lc, void *entry,  in write_metadata()  argument
220 	bio = bio_alloc(lc->logdev->bdev, 1, REQ_OP_WRITE, GFP_KERNEL);  in write_metadata()
225 	bio->bi_private = lc;  in write_metadata()
239 	       lc->sectorsize - entrylen - datalen);  in write_metadata()
242 	ret = bio_add_page(bio, page, lc->sectorsize, 0);  in write_metadata()
243 	if (ret != lc->sectorsize) {  in write_metadata()
253 	put_io_block(lc);  in write_metadata()
257 static int write_inline_data(struct log_writes_c *lc, void *entry,  in write_inline_data()  argument
270 		atomic_inc(&lc->io_blocks);  in write_inline_data()
272 		bio = bio_alloc(lc->logdev->bdev, bio_pages, REQ_OP_WRITE,  in write_inline_data()
277 		bio->bi_private = lc;  in write_inline_data()
281 			pg_sectorlen = ALIGN(pg_datalen, lc->sectorsize);  in write_inline_data()
313 	put_io_block(lc);  in write_inline_data()
317 static int log_one_block(struct log_writes_c *lc,  in log_one_block()  argument
331 	if (write_metadata(lc, &entry, sizeof(entry), block->data,  in log_one_block()
333 		free_pending_block(lc, block);  in log_one_block()
337 	sector += dev_to_bio_sectors(lc, 1);  in log_one_block()
340 		if (write_inline_data(lc, &entry, sizeof(entry), block->data,  in log_one_block()
342 			free_pending_block(lc, block);  in log_one_block()
352 	atomic_inc(&lc->io_blocks);  in log_one_block()
353 	bio = bio_alloc(lc->logdev->bdev, bio_max_segs(block->vec_cnt),  in log_one_block()
358 	bio->bi_private = lc;  in log_one_block()
368 			atomic_inc(&lc->io_blocks);  in log_one_block()
370 			bio = bio_alloc(lc->logdev->bdev,  in log_one_block()
376 			bio->bi_private = lc;  in log_one_block()
392 	put_pending_block(lc);  in log_one_block()
395 	free_pending_block(lc, block);  in log_one_block()
396 	put_io_block(lc);  in log_one_block()
400 static int log_super(struct log_writes_c *lc)  in log_super()  argument
406 	super.nr_entries = cpu_to_le64(lc->logged_entries);  in log_super()
407 	super.sectorsize = cpu_to_le32(lc->sectorsize);  in log_super()
409 	if (write_metadata(lc, &super, sizeof(super), NULL, 0,  in log_super()
419 	wait_for_completion_io(&lc->super_done);  in log_super()
424 static inline sector_t logdev_last_sector(struct log_writes_c *lc)  in logdev_last_sector()  argument
426 	return bdev_nr_sectors(lc->logdev->bdev);  in logdev_last_sector()
431 	struct log_writes_c *lc = (struct log_writes_c *)arg;  in log_writes_kthread()  local
440 		spin_lock_irq(&lc->blocks_lock);  in log_writes_kthread()
441 		if (!list_empty(&lc->logging_blocks)) {  in log_writes_kthread()
442 			block = list_first_entry(&lc->logging_blocks,  in log_writes_kthread()
445 			if (!lc->logging_enabled)  in log_writes_kthread()
448 			sector = lc->next_sector;  in log_writes_kthread()
450 				lc->next_sector += dev_to_bio_sectors(lc, block->nr_sectors);  in log_writes_kthread()
451 			lc->next_sector += dev_to_bio_sectors(lc, 1);  in log_writes_kthread()
457 			if (!lc->end_sector)  in log_writes_kthread()
458 				lc->end_sector = logdev_last_sector(lc);  in log_writes_kthread()
459 			if (lc->end_sector &&  in log_writes_kthread()
460 			    lc->next_sector >= lc->end_sector) {  in log_writes_kthread()
462 				lc->logging_enabled = false;  in log_writes_kthread()
465 			lc->logged_entries++;  in log_writes_kthread()
466 			atomic_inc(&lc->io_blocks);  in log_writes_kthread()
470 				atomic_inc(&lc->io_blocks);  in log_writes_kthread()
473 		logging_enabled = lc->logging_enabled;  in log_writes_kthread()
474 		spin_unlock_irq(&lc->blocks_lock);  in log_writes_kthread()
477 				ret = log_one_block(lc, block, sector);  in log_writes_kthread()
479 					ret = log_super(lc);  in log_writes_kthread()
481 					spin_lock_irq(&lc->blocks_lock);  in log_writes_kthread()
482 					lc->logging_enabled = false;  in log_writes_kthread()
483 					spin_unlock_irq(&lc->blocks_lock);  in log_writes_kthread()
486 				free_pending_block(lc, block);  in log_writes_kthread()
493 			    list_empty(&lc->logging_blocks))  in log_writes_kthread()
507 	struct log_writes_c *lc;  in log_writes_ctr()  local
520 	lc = kzalloc(sizeof(struct log_writes_c), GFP_KERNEL);  in log_writes_ctr()
521 	if (!lc) {  in log_writes_ctr()
525 	spin_lock_init(&lc->blocks_lock);  in log_writes_ctr()
526 	INIT_LIST_HEAD(&lc->unflushed_blocks);  in log_writes_ctr()
527 	INIT_LIST_HEAD(&lc->logging_blocks);  in log_writes_ctr()
528 	init_waitqueue_head(&lc->wait);  in log_writes_ctr()
529 	init_completion(&lc->super_done);  in log_writes_ctr()
530 	atomic_set(&lc->io_blocks, 0);  in log_writes_ctr()
531 	atomic_set(&lc->pending_blocks, 0);  in log_writes_ctr()
534 	ret = dm_get_device(ti, devname, dm_table_get_mode(ti->table), &lc->dev);  in log_writes_ctr()
542 			    &lc->logdev);  in log_writes_ctr()
545 		dm_put_device(ti, lc->dev);  in log_writes_ctr()
549 	lc->sectorsize = bdev_logical_block_size(lc->dev->bdev);  in log_writes_ctr()
550 	lc->sectorshift = ilog2(lc->sectorsize);  in log_writes_ctr()
551 	lc->log_kthread = kthread_run(log_writes_kthread, lc, "log-write");  in log_writes_ctr()
552 	if (IS_ERR(lc->log_kthread)) {  in log_writes_ctr()
553 		ret = PTR_ERR(lc->log_kthread);  in log_writes_ctr()
555 		dm_put_device(ti, lc->dev);  in log_writes_ctr()
556 		dm_put_device(ti, lc->logdev);  in log_writes_ctr()
565 	lc->next_sector = lc->sectorsize >> SECTOR_SHIFT;  in log_writes_ctr()
566 	lc->logging_enabled = true;  in log_writes_ctr()
567 	lc->end_sector = logdev_last_sector(lc);  in log_writes_ctr()
568 	lc->device_supports_discard = true;  in log_writes_ctr()
575 	ti->private = lc;  in log_writes_ctr()
579 	kfree(lc);  in log_writes_ctr()
583 static int log_mark(struct log_writes_c *lc, char *data)  in log_mark()  argument
586 	size_t maxsize = lc->sectorsize - sizeof(struct log_write_entry);  in log_mark()
600 	atomic_inc(&lc->pending_blocks);  in log_mark()
603 	spin_lock_irq(&lc->blocks_lock);  in log_mark()
604 	list_add_tail(&block->list, &lc->logging_blocks);  in log_mark()
605 	spin_unlock_irq(&lc->blocks_lock);  in log_mark()
606 	wake_up_process(lc->log_kthread);  in log_mark()
612 	struct log_writes_c *lc = ti->private;  in log_writes_dtr()  local
614 	spin_lock_irq(&lc->blocks_lock);  in log_writes_dtr()
615 	list_splice_init(&lc->unflushed_blocks, &lc->logging_blocks);  in log_writes_dtr()
616 	spin_unlock_irq(&lc->blocks_lock);  in log_writes_dtr()
622 	log_mark(lc, "dm-log-writes-end");  in log_writes_dtr()
623 	wake_up_process(lc->log_kthread);  in log_writes_dtr()
624 	wait_event(lc->wait, !atomic_read(&lc->io_blocks) &&  in log_writes_dtr()
625 		   !atomic_read(&lc->pending_blocks));  in log_writes_dtr()
626 	kthread_stop(lc->log_kthread);  in log_writes_dtr()
628 	WARN_ON(!list_empty(&lc->logging_blocks));  in log_writes_dtr()
629 	WARN_ON(!list_empty(&lc->unflushed_blocks));  in log_writes_dtr()
630 	dm_put_device(ti, lc->dev);  in log_writes_dtr()
631 	dm_put_device(ti, lc->logdev);  in log_writes_dtr()
632 	kfree(lc);  in log_writes_dtr()
637 	struct log_writes_c *lc = ti->private;  in normal_map_bio()  local
639 	bio_set_dev(bio, lc->dev->bdev);  in normal_map_bio()
644 	struct log_writes_c *lc = ti->private;  in log_writes_map()  local
659 	if (!lc->logging_enabled)  in log_writes_map()
684 		spin_lock_irq(&lc->blocks_lock);  in log_writes_map()
685 		lc->logging_enabled = false;  in log_writes_map()
686 		spin_unlock_irq(&lc->blocks_lock);  in log_writes_map()
691 	atomic_inc(&lc->pending_blocks);  in log_writes_map()
702 	block->sector = bio_to_dev_sectors(lc, bio->bi_iter.bi_sector);  in log_writes_map()
703 	block->nr_sectors = bio_to_dev_sectors(lc, bio_sectors(bio));  in log_writes_map()
708 		if (lc->device_supports_discard)  in log_writes_map()
716 		spin_lock_irq(&lc->blocks_lock);  in log_writes_map()
717 		list_splice_init(&lc->unflushed_blocks, &block->list);  in log_writes_map()
718 		spin_unlock_irq(&lc->blocks_lock);  in log_writes_map()
738 			free_pending_block(lc, block);  in log_writes_map()
739 			spin_lock_irq(&lc->blocks_lock);  in log_writes_map()
740 			lc->logging_enabled = false;  in log_writes_map()
741 			spin_unlock_irq(&lc->blocks_lock);  in log_writes_map()
756 		spin_lock_irq(&lc->blocks_lock);  in log_writes_map()
757 		list_splice_init(&lc->unflushed_blocks, &block->list);  in log_writes_map()
758 		spin_unlock_irq(&lc->blocks_lock);  in log_writes_map()
768 	struct log_writes_c *lc = ti->private;  in normal_end_io()  local
775 		spin_lock_irqsave(&lc->blocks_lock, flags);  in normal_end_io()
777 			list_splice_tail_init(&block->list, &lc->logging_blocks);  in normal_end_io()
778 			list_add_tail(&block->list, &lc->logging_blocks);  in normal_end_io()
779 			wake_up_process(lc->log_kthread);  in normal_end_io()
781 			list_add_tail(&block->list, &lc->logging_blocks);  in normal_end_io()
782 			wake_up_process(lc->log_kthread);  in normal_end_io()
784 			list_add_tail(&block->list, &lc->unflushed_blocks);  in normal_end_io()
785 		spin_unlock_irqrestore(&lc->blocks_lock, flags);  in normal_end_io()
799 	struct log_writes_c *lc = ti->private;  in log_writes_status()  local
803 		DMEMIT("%llu %llu", lc->logged_entries,  in log_writes_status()
804 		       (unsigned long long)lc->next_sector - 1);  in log_writes_status()
805 		if (!lc->logging_enabled)  in log_writes_status()
810 		DMEMIT("%s %s", lc->dev->name, lc->logdev->name);  in log_writes_status()
822 	struct log_writes_c *lc = ti->private;  in log_writes_prepare_ioctl()  local
823 	struct dm_dev *dev = lc->dev;  in log_writes_prepare_ioctl()
838 	struct log_writes_c *lc = ti->private;  in log_writes_iterate_devices()  local
840 	return fn(ti, lc->dev, 0, ti->len, data);  in log_writes_iterate_devices()
851 	struct log_writes_c *lc = ti->private;  in log_writes_message()  local
859 		r = log_mark(lc, argv[1]);  in log_writes_message()
868 	struct log_writes_c *lc = ti->private;  in log_writes_io_hints()  local
870 	if (!bdev_max_discard_sectors(lc->dev->bdev)) {  in log_writes_io_hints()
871 		lc->device_supports_discard = false;  in log_writes_io_hints()
872 		limits->discard_granularity = lc->sectorsize;  in log_writes_io_hints()
875 	limits->logical_block_size = bdev_logical_block_size(lc->dev->bdev);  in log_writes_io_hints()
876 	limits->physical_block_size = bdev_physical_block_size(lc->dev->bdev);  in log_writes_io_hints()
885 	struct log_writes_c *lc = ti->private;  in log_writes_dax_pgoff()  local
887 	*pgoff += (get_start_sect(lc->dev->bdev) >> PAGE_SECTORS_SHIFT);  in log_writes_dax_pgoff()
888 	return lc->dev->dax_dev;  in log_writes_dax_pgoff()