33 	struct block_device bdev;  member
44 	return &BDEV_I(inode)->bdev;  in I_BDEV()
48 static void bdev_write_inode(struct block_device *bdev)  in bdev_write_inode()  argument
50 	struct inode *inode = bdev->bd_inode;  in bdev_write_inode()
61 					    bdevname(bdev, name), ret);  in bdev_write_inode()
69 static void kill_bdev(struct block_device *bdev)  in kill_bdev()  argument
71 	struct address_space *mapping = bdev->bd_inode->i_mapping;  in kill_bdev()
81 void invalidate_bdev(struct block_device *bdev)  in invalidate_bdev()  argument
83 	struct address_space *mapping = bdev->bd_inode->i_mapping;  in invalidate_bdev()
101 int truncate_bdev_range(struct block_device *bdev, fmode_t mode,  in truncate_bdev_range()  argument
110 		int err = bd_prepare_to_claim(bdev, truncate_bdev_range);  in truncate_bdev_range()
115 	truncate_inode_pages_range(bdev->bd_inode->i_mapping, lstart, lend);  in truncate_bdev_range()
117 		bd_abort_claiming(bdev, truncate_bdev_range);  in truncate_bdev_range()
125 	return invalidate_inode_pages2_range(bdev->bd_inode->i_mapping,  in truncate_bdev_range()
130 static void set_init_blocksize(struct block_device *bdev)  in set_init_blocksize()  argument
132 	unsigned int bsize = bdev_logical_block_size(bdev);  in set_init_blocksize()
133 	loff_t size = i_size_read(bdev->bd_inode);  in set_init_blocksize()
140 	bdev->bd_inode->i_blkbits = blksize_bits(bsize);  in set_init_blocksize()
143 int set_blocksize(struct block_device *bdev, int size)  in set_blocksize()  argument
150 	if (size < bdev_logical_block_size(bdev))  in set_blocksize()
154 	if (bdev->bd_inode->i_blkbits != blksize_bits(size)) {  in set_blocksize()
155 		sync_blockdev(bdev);  in set_blocksize()
156 		bdev->bd_inode->i_blkbits = blksize_bits(size);  in set_blocksize()
157 		kill_bdev(bdev);  in set_blocksize()
187 int __sync_blockdev(struct block_device *bdev, int wait)  in __sync_blockdev()  argument
189 	if (!bdev)  in __sync_blockdev()
192 		return filemap_flush(bdev->bd_inode->i_mapping);  in __sync_blockdev()
193 	return filemap_write_and_wait(bdev->bd_inode->i_mapping);  in __sync_blockdev()
200 int sync_blockdev(struct block_device *bdev)  in sync_blockdev()  argument
202 	return __sync_blockdev(bdev, 1);  in sync_blockdev()
211 int fsync_bdev(struct block_device *bdev)  in fsync_bdev()  argument
213 	struct super_block *sb = get_super(bdev);  in fsync_bdev()
219 	return sync_blockdev(bdev);  in fsync_bdev()
235 int freeze_bdev(struct block_device *bdev)  in freeze_bdev()  argument
240 	mutex_lock(&bdev->bd_fsfreeze_mutex);  in freeze_bdev()
241 	if (++bdev->bd_fsfreeze_count > 1)  in freeze_bdev()
244 	sb = get_active_super(bdev);  in freeze_bdev()
254 		bdev->bd_fsfreeze_count--;  in freeze_bdev()
257 	bdev->bd_fsfreeze_sb = sb;  in freeze_bdev()
260 	sync_blockdev(bdev);  in freeze_bdev()
262 	mutex_unlock(&bdev->bd_fsfreeze_mutex);  in freeze_bdev()
273 int thaw_bdev(struct block_device *bdev)  in thaw_bdev()  argument
278 	mutex_lock(&bdev->bd_fsfreeze_mutex);  in thaw_bdev()
279 	if (!bdev->bd_fsfreeze_count)  in thaw_bdev()
283 	if (--bdev->bd_fsfreeze_count > 0)  in thaw_bdev()
286 	sb = bdev->bd_fsfreeze_sb;  in thaw_bdev()
295 		bdev->bd_fsfreeze_count++;  in thaw_bdev()
297 		bdev->bd_fsfreeze_sb = NULL;  in thaw_bdev()
299 	mutex_unlock(&bdev->bd_fsfreeze_mutex);  in thaw_bdev()
320 int bdev_read_page(struct block_device *bdev, sector_t sector,  in bdev_read_page()  argument
323 	const struct block_device_operations *ops = bdev->bd_disk->fops;  in bdev_read_page()
326 	if (!ops->rw_page || bdev_get_integrity(bdev))  in bdev_read_page()
329 	result = blk_queue_enter(bdev->bd_disk->queue, 0);  in bdev_read_page()
332 	result = ops->rw_page(bdev, sector + get_start_sect(bdev), page,  in bdev_read_page()
334 	blk_queue_exit(bdev->bd_disk->queue);  in bdev_read_page()
357 int bdev_write_page(struct block_device *bdev, sector_t sector,  in bdev_write_page()  argument
361 	const struct block_device_operations *ops = bdev->bd_disk->fops;  in bdev_write_page()
363 	if (!ops->rw_page || bdev_get_integrity(bdev))  in bdev_write_page()
365 	result = blk_queue_enter(bdev->bd_disk->queue, 0);  in bdev_write_page()
370 	result = ops->rw_page(bdev, sector + get_start_sect(bdev), page,  in bdev_write_page()
378 	blk_queue_exit(bdev->bd_disk->queue);  in bdev_write_page()
395 	memset(&ei->bdev, 0, sizeof(ei->bdev));  in bdev_alloc_inode()
401 	struct block_device *bdev = I_BDEV(inode);  in bdev_free_inode()  local
403 	free_percpu(bdev->bd_stats);  in bdev_free_inode()
404 	kfree(bdev->bd_meta_info);  in bdev_free_inode()
406 	if (!bdev_is_partition(bdev)) {  in bdev_free_inode()
407 		if (bdev->bd_disk && bdev->bd_disk->bdi)  in bdev_free_inode()
408 			bdi_put(bdev->bd_disk->bdi);  in bdev_free_inode()
409 		kfree(bdev->bd_disk);  in bdev_free_inode()
412 	if (MAJOR(bdev->bd_dev) == BLOCK_EXT_MAJOR)  in bdev_free_inode()
413 		blk_free_ext_minor(MINOR(bdev->bd_dev));  in bdev_free_inode()
479 	struct block_device *bdev;  in bdev_alloc()  local
490 	bdev = I_BDEV(inode);  in bdev_alloc()
491 	mutex_init(&bdev->bd_fsfreeze_mutex);  in bdev_alloc()
492 	spin_lock_init(&bdev->bd_size_lock);  in bdev_alloc()
493 	bdev->bd_partno = partno;  in bdev_alloc()
494 	bdev->bd_inode = inode;  in bdev_alloc()
495 	bdev->bd_stats = alloc_percpu(struct disk_stats);  in bdev_alloc()
496 	if (!bdev->bd_stats) {  in bdev_alloc()
500 	bdev->bd_disk = disk;  in bdev_alloc()
501 	return bdev;  in bdev_alloc()
504 void bdev_add(struct block_device *bdev, dev_t dev)  in bdev_add()  argument
506 	bdev->bd_dev = dev;  in bdev_add()
507 	bdev->bd_inode->i_rdev = dev;  in bdev_add()
508 	bdev->bd_inode->i_ino = dev;  in bdev_add()
509 	insert_inode_hash(bdev->bd_inode);  in bdev_add()
539 static bool bd_may_claim(struct block_device *bdev, struct block_device *whole,  in bd_may_claim()  argument
542 	if (bdev->bd_holder == holder)  in bd_may_claim()
544 	else if (bdev->bd_holder != NULL)  in bd_may_claim()
546 	else if (whole == bdev)  in bd_may_claim()
569 int bd_prepare_to_claim(struct block_device *bdev, void *holder)  in bd_prepare_to_claim()  argument
571 	struct block_device *whole = bdev_whole(bdev);  in bd_prepare_to_claim()
578 	if (!bd_may_claim(bdev, whole, holder)) {  in bd_prepare_to_claim()
619 static void bd_finish_claiming(struct block_device *bdev, void *holder)  in bd_finish_claiming()  argument
621 	struct block_device *whole = bdev_whole(bdev);  in bd_finish_claiming()
624 	BUG_ON(!bd_may_claim(bdev, whole, holder));  in bd_finish_claiming()
631 	bdev->bd_holders++;  in bd_finish_claiming()
632 	bdev->bd_holder = holder;  in bd_finish_claiming()
646 void bd_abort_claiming(struct block_device *bdev, void *holder)  in bd_abort_claiming()  argument
649 	bd_clear_claiming(bdev_whole(bdev), holder);  in bd_abort_claiming()
654 static void blkdev_flush_mapping(struct block_device *bdev)  in blkdev_flush_mapping()  argument
656 	WARN_ON_ONCE(bdev->bd_holders);  in blkdev_flush_mapping()
657 	sync_blockdev(bdev);  in blkdev_flush_mapping()
658 	kill_bdev(bdev);  in blkdev_flush_mapping()
659 	bdev_write_inode(bdev);  in blkdev_flush_mapping()
662 static int blkdev_get_whole(struct block_device *bdev, fmode_t mode)  in blkdev_get_whole()  argument
664 	struct gendisk *disk = bdev->bd_disk;  in blkdev_get_whole()
668 		ret = disk->fops->open(bdev, mode);  in blkdev_get_whole()
678 	if (!bdev->bd_openers)  in blkdev_get_whole()
679 		set_init_blocksize(bdev);  in blkdev_get_whole()
682 	bdev->bd_openers++;  in blkdev_get_whole()
686 static void blkdev_put_whole(struct block_device *bdev, fmode_t mode)  in blkdev_put_whole()  argument
688 	if (!--bdev->bd_openers)  in blkdev_put_whole()
689 		blkdev_flush_mapping(bdev);  in blkdev_put_whole()
690 	if (bdev->bd_disk->fops->release)  in blkdev_put_whole()
691 		bdev->bd_disk->fops->release(bdev->bd_disk, mode);  in blkdev_put_whole()
734 	struct block_device *bdev;  in blkdev_get_no_open()  local
746 	bdev = &BDEV_I(inode)->bdev;  in blkdev_get_no_open()
747 	if (!kobject_get_unless_zero(&bdev->bd_device.kobj))  in blkdev_get_no_open()
748 		bdev = NULL;  in blkdev_get_no_open()
751 	if (!bdev)  in blkdev_get_no_open()
753 	if ((bdev->bd_disk->flags & GENHD_FL_HIDDEN) ||  in blkdev_get_no_open()
754 	    !try_module_get(bdev->bd_disk->fops->owner)) {  in blkdev_get_no_open()
755 		put_device(&bdev->bd_device);  in blkdev_get_no_open()
759 	return bdev;  in blkdev_get_no_open()
762 void blkdev_put_no_open(struct block_device *bdev)  in blkdev_put_no_open()  argument
764 	module_put(bdev->bd_disk->fops->owner);  in blkdev_put_no_open()
765 	put_device(&bdev->bd_device);  in blkdev_put_no_open()
792 	struct block_device *bdev;  in blkdev_get_by_dev()  local
803 	bdev = blkdev_get_no_open(dev);  in blkdev_get_by_dev()
804 	if (!bdev)  in blkdev_get_by_dev()
806 	disk = bdev->bd_disk;  in blkdev_get_by_dev()
809 		ret = bd_prepare_to_claim(bdev, holder);  in blkdev_get_by_dev()
820 	if (bdev_is_partition(bdev))  in blkdev_get_by_dev()
821 		ret = blkdev_get_part(bdev, mode);  in blkdev_get_by_dev()
823 		ret = blkdev_get_whole(bdev, mode);  in blkdev_get_by_dev()
827 		bd_finish_claiming(bdev, holder);  in blkdev_get_by_dev()
836 		if ((mode & FMODE_WRITE) && !bdev->bd_write_holder &&  in blkdev_get_by_dev()
838 			bdev->bd_write_holder = true;  in blkdev_get_by_dev()
846 	return bdev;  in blkdev_get_by_dev()
850 		bd_abort_claiming(bdev, holder);  in blkdev_get_by_dev()
854 	blkdev_put_no_open(bdev);  in blkdev_get_by_dev()
879 	struct block_device *bdev;  in blkdev_get_by_path()  local
887 	bdev = blkdev_get_by_dev(dev, mode, holder);  in blkdev_get_by_path()
888 	if (!IS_ERR(bdev) && (mode & FMODE_WRITE) && bdev_read_only(bdev)) {  in blkdev_get_by_path()
889 		blkdev_put(bdev, mode);  in blkdev_get_by_path()
893 	return bdev;  in blkdev_get_by_path()
897 void blkdev_put(struct block_device *bdev, fmode_t mode)  in blkdev_put()  argument
899 	struct gendisk *disk = bdev->bd_disk;  in blkdev_put()
908 	if (bdev->bd_openers == 1)  in blkdev_put()
909 		sync_blockdev(bdev);  in blkdev_put()
913 		struct block_device *whole = bdev_whole(bdev);  in blkdev_put()
923 		WARN_ON_ONCE(--bdev->bd_holders < 0);  in blkdev_put()
926 		if ((bdev_free = !bdev->bd_holders))  in blkdev_put()
927 			bdev->bd_holder = NULL;  in blkdev_put()
937 		if (bdev_free && bdev->bd_write_holder) {  in blkdev_put()
939 			bdev->bd_write_holder = false;  in blkdev_put()
950 	if (bdev_is_partition(bdev))  in blkdev_put()
951 		blkdev_put_part(bdev, mode);  in blkdev_put()
953 		blkdev_put_whole(bdev, mode);  in blkdev_put()
956 	blkdev_put_no_open(bdev);  in blkdev_put()
998 int __invalidate_device(struct block_device *bdev, bool kill_dirty)  in __invalidate_device()  argument
1000 	struct super_block *sb = get_super(bdev);  in __invalidate_device()
1014 	invalidate_bdev(bdev);  in __invalidate_device()
1026 		struct block_device *bdev;  in iterate_bdevs()  local
1047 		bdev = I_BDEV(inode);  in iterate_bdevs()
1049 		mutex_lock(&bdev->bd_disk->open_mutex);  in iterate_bdevs()
1050 		if (bdev->bd_openers)  in iterate_bdevs()
1051 			func(bdev, arg);  in iterate_bdevs()
1052 		mutex_unlock(&bdev->bd_disk->open_mutex);  in iterate_bdevs()