135 static inline sector_t bio_to_dev_sectors(struct log_writes_c *lc,  in bio_to_dev_sectors()  argument
138 	return sectors >> (lc->sectorshift - SECTOR_SHIFT);  in bio_to_dev_sectors()
141 static inline sector_t dev_to_bio_sectors(struct log_writes_c *lc,  in dev_to_bio_sectors()  argument
144 	return sectors << (lc->sectorshift - SECTOR_SHIFT);  in dev_to_bio_sectors()
147 static void put_pending_block(struct log_writes_c *lc)  in put_pending_block()  argument
149 	if (atomic_dec_and_test(&lc->pending_blocks)) {  in put_pending_block()
151 		if (waitqueue_active(&lc->wait))  in put_pending_block()
152 			wake_up(&lc->wait);  in put_pending_block()
156 static void put_io_block(struct log_writes_c *lc)  in put_io_block()  argument
158 	if (atomic_dec_and_test(&lc->io_blocks)) {  in put_io_block()
160 		if (waitqueue_active(&lc->wait))  in put_io_block()
161 			wake_up(&lc->wait);  in put_io_block()
167 	struct log_writes_c *lc = bio->bi_private;  in log_end_io()  local
173 		spin_lock_irqsave(&lc->blocks_lock, flags);  in log_end_io()
174 		lc->logging_enabled = false;  in log_end_io()
175 		spin_unlock_irqrestore(&lc->blocks_lock, flags);  in log_end_io()
179 	put_io_block(lc);  in log_end_io()
187 static void free_pending_block(struct log_writes_c *lc,  in free_pending_block()  argument
198 	put_pending_block(lc);  in free_pending_block()
201 static int write_metadata(struct log_writes_c *lc, void *entry,  in write_metadata()  argument
217 	bio_set_dev(bio, lc->logdev->bdev);  in write_metadata()
219 	bio->bi_private = lc;  in write_metadata()
234 	       lc->sectorsize - entrylen - datalen);  in write_metadata()
237 	ret = bio_add_page(bio, page, lc->sectorsize, 0);  in write_metadata()
238 	if (ret != lc->sectorsize) {  in write_metadata()
248 	put_io_block(lc);  in write_metadata()
252 static int write_inline_data(struct log_writes_c *lc, void *entry,  in write_inline_data()  argument
266 		atomic_inc(&lc->io_blocks);  in write_inline_data()
276 		bio_set_dev(bio, lc->logdev->bdev);  in write_inline_data()
278 		bio->bi_private = lc;  in write_inline_data()
283 			pg_sectorlen = ALIGN(pg_datalen, lc->sectorsize);  in write_inline_data()
316 	put_io_block(lc);  in write_inline_data()
320 static int log_one_block(struct log_writes_c *lc,  in log_one_block()  argument
334 	if (write_metadata(lc, &entry, sizeof(entry), block->data,  in log_one_block()
336 		free_pending_block(lc, block);  in log_one_block()
340 	sector += dev_to_bio_sectors(lc, 1);  in log_one_block()
343 		if (write_inline_data(lc, &entry, sizeof(entry), block->data,  in log_one_block()
345 			free_pending_block(lc, block);  in log_one_block()
355 	atomic_inc(&lc->io_blocks);  in log_one_block()
363 	bio_set_dev(bio, lc->logdev->bdev);  in log_one_block()
365 	bio->bi_private = lc;  in log_one_block()
376 			atomic_inc(&lc->io_blocks);  in log_one_block()
385 			bio_set_dev(bio, lc->logdev->bdev);  in log_one_block()
387 			bio->bi_private = lc;  in log_one_block()
404 	put_pending_block(lc);  in log_one_block()
407 	free_pending_block(lc, block);  in log_one_block()
408 	put_io_block(lc);  in log_one_block()
412 static int log_super(struct log_writes_c *lc)  in log_super()  argument
418 	super.nr_entries = cpu_to_le64(lc->logged_entries);  in log_super()
419 	super.sectorsize = cpu_to_le32(lc->sectorsize);  in log_super()
421 	if (write_metadata(lc, &super, sizeof(super), NULL, 0, 0)) {  in log_super()
429 static inline sector_t logdev_last_sector(struct log_writes_c *lc)  in logdev_last_sector()  argument
431 	return i_size_read(lc->logdev->bdev->bd_inode) >> SECTOR_SHIFT;  in logdev_last_sector()
436 	struct log_writes_c *lc = (struct log_writes_c *)arg;  in log_writes_kthread()  local
445 		spin_lock_irq(&lc->blocks_lock);  in log_writes_kthread()
446 		if (!list_empty(&lc->logging_blocks)) {  in log_writes_kthread()
447 			block = list_first_entry(&lc->logging_blocks,  in log_writes_kthread()
450 			if (!lc->logging_enabled)  in log_writes_kthread()
453 			sector = lc->next_sector;  in log_writes_kthread()
455 				lc->next_sector += dev_to_bio_sectors(lc, block->nr_sectors);  in log_writes_kthread()
456 			lc->next_sector += dev_to_bio_sectors(lc, 1);  in log_writes_kthread()
462 			if (!lc->end_sector)  in log_writes_kthread()
463 				lc->end_sector = logdev_last_sector(lc);  in log_writes_kthread()
464 			if (lc->end_sector &&  in log_writes_kthread()
465 			    lc->next_sector >= lc->end_sector) {  in log_writes_kthread()
467 				lc->logging_enabled = false;  in log_writes_kthread()
470 			lc->logged_entries++;  in log_writes_kthread()
471 			atomic_inc(&lc->io_blocks);  in log_writes_kthread()
475 				atomic_inc(&lc->io_blocks);  in log_writes_kthread()
478 		logging_enabled = lc->logging_enabled;  in log_writes_kthread()
479 		spin_unlock_irq(&lc->blocks_lock);  in log_writes_kthread()
482 				ret = log_one_block(lc, block, sector);  in log_writes_kthread()
484 					ret = log_super(lc);  in log_writes_kthread()
486 					spin_lock_irq(&lc->blocks_lock);  in log_writes_kthread()
487 					lc->logging_enabled = false;  in log_writes_kthread()
488 					spin_unlock_irq(&lc->blocks_lock);  in log_writes_kthread()
491 				free_pending_block(lc, block);  in log_writes_kthread()
498 			    list_empty(&lc->logging_blocks))  in log_writes_kthread()
512 	struct log_writes_c *lc;  in log_writes_ctr()  local
525 	lc = kzalloc(sizeof(struct log_writes_c), GFP_KERNEL);  in log_writes_ctr()
526 	if (!lc) {  in log_writes_ctr()
530 	spin_lock_init(&lc->blocks_lock);  in log_writes_ctr()
531 	INIT_LIST_HEAD(&lc->unflushed_blocks);  in log_writes_ctr()
532 	INIT_LIST_HEAD(&lc->logging_blocks);  in log_writes_ctr()
533 	init_waitqueue_head(&lc->wait);  in log_writes_ctr()
534 	atomic_set(&lc->io_blocks, 0);  in log_writes_ctr()
535 	atomic_set(&lc->pending_blocks, 0);  in log_writes_ctr()
538 	ret = dm_get_device(ti, devname, dm_table_get_mode(ti->table), &lc->dev);  in log_writes_ctr()
546 			    &lc->logdev);  in log_writes_ctr()
549 		dm_put_device(ti, lc->dev);  in log_writes_ctr()
553 	lc->sectorsize = bdev_logical_block_size(lc->dev->bdev);  in log_writes_ctr()
554 	lc->sectorshift = ilog2(lc->sectorsize);  in log_writes_ctr()
555 	lc->log_kthread = kthread_run(log_writes_kthread, lc, "log-write");  in log_writes_ctr()
556 	if (IS_ERR(lc->log_kthread)) {  in log_writes_ctr()
557 		ret = PTR_ERR(lc->log_kthread);  in log_writes_ctr()
559 		dm_put_device(ti, lc->dev);  in log_writes_ctr()
560 		dm_put_device(ti, lc->logdev);  in log_writes_ctr()
569 	lc->next_sector = lc->sectorsize >> SECTOR_SHIFT;  in log_writes_ctr()
570 	lc->logging_enabled = true;  in log_writes_ctr()
571 	lc->end_sector = logdev_last_sector(lc);  in log_writes_ctr()
572 	lc->device_supports_discard = true;  in log_writes_ctr()
579 	ti->private = lc;  in log_writes_ctr()
583 	kfree(lc);  in log_writes_ctr()
587 static int log_mark(struct log_writes_c *lc, char *data)  in log_mark()  argument
590 	size_t maxsize = lc->sectorsize - sizeof(struct log_write_entry);  in log_mark()
604 	atomic_inc(&lc->pending_blocks);  in log_mark()
607 	spin_lock_irq(&lc->blocks_lock);  in log_mark()
608 	list_add_tail(&block->list, &lc->logging_blocks);  in log_mark()
609 	spin_unlock_irq(&lc->blocks_lock);  in log_mark()
610 	wake_up_process(lc->log_kthread);  in log_mark()
616 	struct log_writes_c *lc = ti->private;  in log_writes_dtr()  local
618 	spin_lock_irq(&lc->blocks_lock);  in log_writes_dtr()
619 	list_splice_init(&lc->unflushed_blocks, &lc->logging_blocks);  in log_writes_dtr()
620 	spin_unlock_irq(&lc->blocks_lock);  in log_writes_dtr()
626 	log_mark(lc, "dm-log-writes-end");  in log_writes_dtr()
627 	wake_up_process(lc->log_kthread);  in log_writes_dtr()
628 	wait_event(lc->wait, !atomic_read(&lc->io_blocks) &&  in log_writes_dtr()
629 		   !atomic_read(&lc->pending_blocks));  in log_writes_dtr()
630 	kthread_stop(lc->log_kthread);  in log_writes_dtr()
632 	WARN_ON(!list_empty(&lc->logging_blocks));  in log_writes_dtr()
633 	WARN_ON(!list_empty(&lc->unflushed_blocks));  in log_writes_dtr()
634 	dm_put_device(ti, lc->dev);  in log_writes_dtr()
635 	dm_put_device(ti, lc->logdev);  in log_writes_dtr()
636 	kfree(lc);  in log_writes_dtr()
641 	struct log_writes_c *lc = ti->private;  in normal_map_bio()  local
643 	bio_set_dev(bio, lc->dev->bdev);  in normal_map_bio()
648 	struct log_writes_c *lc = ti->private;  in log_writes_map()  local
663 	if (!lc->logging_enabled)  in log_writes_map()
688 		spin_lock_irq(&lc->blocks_lock);  in log_writes_map()
689 		lc->logging_enabled = false;  in log_writes_map()
690 		spin_unlock_irq(&lc->blocks_lock);  in log_writes_map()
695 	atomic_inc(&lc->pending_blocks);  in log_writes_map()
706 	block->sector = bio_to_dev_sectors(lc, bio->bi_iter.bi_sector);  in log_writes_map()
707 	block->nr_sectors = bio_to_dev_sectors(lc, bio_sectors(bio));  in log_writes_map()
712 		if (lc->device_supports_discard)  in log_writes_map()
720 		spin_lock_irq(&lc->blocks_lock);  in log_writes_map()
721 		list_splice_init(&lc->unflushed_blocks, &block->list);  in log_writes_map()
722 		spin_unlock_irq(&lc->blocks_lock);  in log_writes_map()
742 			free_pending_block(lc, block);  in log_writes_map()
743 			spin_lock_irq(&lc->blocks_lock);  in log_writes_map()
744 			lc->logging_enabled = false;  in log_writes_map()
745 			spin_unlock_irq(&lc->blocks_lock);  in log_writes_map()
762 		spin_lock_irq(&lc->blocks_lock);  in log_writes_map()
763 		list_splice_init(&lc->unflushed_blocks, &block->list);  in log_writes_map()
764 		spin_unlock_irq(&lc->blocks_lock);  in log_writes_map()
774 	struct log_writes_c *lc = ti->private;  in normal_end_io()  local
781 		spin_lock_irqsave(&lc->blocks_lock, flags);  in normal_end_io()
783 			list_splice_tail_init(&block->list, &lc->logging_blocks);  in normal_end_io()
784 			list_add_tail(&block->list, &lc->logging_blocks);  in normal_end_io()
785 			wake_up_process(lc->log_kthread);  in normal_end_io()
787 			list_add_tail(&block->list, &lc->logging_blocks);  in normal_end_io()
788 			wake_up_process(lc->log_kthread);  in normal_end_io()
790 			list_add_tail(&block->list, &lc->unflushed_blocks);  in normal_end_io()
791 		spin_unlock_irqrestore(&lc->blocks_lock, flags);  in normal_end_io()
805 	struct log_writes_c *lc = ti->private;  in log_writes_status()  local
809 		DMEMIT("%llu %llu", lc->logged_entries,  in log_writes_status()
810 		       (unsigned long long)lc->next_sector - 1);  in log_writes_status()
811 		if (!lc->logging_enabled)  in log_writes_status()
816 		DMEMIT("%s %s", lc->dev->name, lc->logdev->name);  in log_writes_status()
824 	struct log_writes_c *lc = ti->private;  in log_writes_prepare_ioctl()  local
825 	struct dm_dev *dev = lc->dev;  in log_writes_prepare_ioctl()
840 	struct log_writes_c *lc = ti->private;  in log_writes_iterate_devices()  local
842 	return fn(ti, lc->dev, 0, ti->len, data);  in log_writes_iterate_devices()
853 	struct log_writes_c *lc = ti->private;  in log_writes_message()  local
861 		r = log_mark(lc, argv[1]);  in log_writes_message()
870 	struct log_writes_c *lc = ti->private;  in log_writes_io_hints()  local
871 	struct request_queue *q = bdev_get_queue(lc->dev->bdev);  in log_writes_io_hints()
874 		lc->device_supports_discard = false;  in log_writes_io_hints()
875 		limits->discard_granularity = lc->sectorsize;  in log_writes_io_hints()
878 	limits->logical_block_size = bdev_logical_block_size(lc->dev->bdev);  in log_writes_io_hints()
879 	limits->physical_block_size = bdev_physical_block_size(lc->dev->bdev);  in log_writes_io_hints()
884 static int log_dax(struct log_writes_c *lc, sector_t sector, size_t bytes,  in log_dax()  argument
917 	block->sector = bio_to_dev_sectors(lc, sector);  in log_dax()
918 	block->nr_sectors = ALIGN(bytes, lc->sectorsize) >> lc->sectorshift;  in log_dax()
920 	atomic_inc(&lc->pending_blocks);  in log_dax()
921 	spin_lock_irq(&lc->blocks_lock);  in log_dax()
922 	list_add_tail(&block->list, &lc->unflushed_blocks);  in log_dax()
923 	spin_unlock_irq(&lc->blocks_lock);  in log_dax()
924 	wake_up_process(lc->log_kthread);  in log_dax()
932 	struct log_writes_c *lc = ti->private;  in log_writes_dax_direct_access()  local
936 	ret = bdev_dax_pgoff(lc->dev->bdev, sector, nr_pages * PAGE_SIZE, &pgoff);  in log_writes_dax_direct_access()
939 	return dax_direct_access(lc->dev->dax_dev, pgoff, nr_pages, kaddr, pfn);  in log_writes_dax_direct_access()
946 	struct log_writes_c *lc = ti->private;  in log_writes_dax_copy_from_iter()  local
950 	if (bdev_dax_pgoff(lc->dev->bdev, sector, ALIGN(bytes, PAGE_SIZE), &pgoff))  in log_writes_dax_copy_from_iter()
954 	if (!lc->logging_enabled)  in log_writes_dax_copy_from_iter()
957 	err = log_dax(lc, sector, bytes, i);  in log_writes_dax_copy_from_iter()
963 	return dax_copy_from_iter(lc->dev->dax_dev, pgoff, addr, bytes, i);  in log_writes_dax_copy_from_iter()
970 	struct log_writes_c *lc = ti->private;  in log_writes_dax_copy_to_iter()  local
973 	if (bdev_dax_pgoff(lc->dev->bdev, sector, ALIGN(bytes, PAGE_SIZE), &pgoff))  in log_writes_dax_copy_to_iter()
975 	return dax_copy_to_iter(lc->dev->dax_dev, pgoff, addr, bytes, i);  in log_writes_dax_copy_to_iter()