69 static void trace_note(struct blk_trace *bt, pid_t pid, int action,  in trace_note()  argument
92 	if (!bt->rchan)  in trace_note()
95 	t = relay_reserve(bt->rchan, sizeof(*t) + len + cgid_len);  in trace_note()
100 		t->device = bt->dev;  in trace_note()
121 	struct blk_trace *bt;  in trace_note_tsk()  local
125 	list_for_each_entry(bt, &running_trace_list, running_list) {  in trace_note_tsk()
126 		trace_note(bt, tsk->pid, BLK_TN_PROCESS, tsk->comm,  in trace_note_tsk()
132 static void trace_note_time(struct blk_trace *bt)  in trace_note_time()  argument
144 	trace_note(bt, 0, BLK_TN_TIMESTAMP, words, sizeof(words), 0);  in trace_note_time()
148 void __trace_note_message(struct blk_trace *bt, struct blkcg *blkcg,  in __trace_note_message()  argument
156 	if (unlikely(bt->trace_state != Blktrace_running &&  in __trace_note_message()
164 	if (!(bt->act_mask & BLK_TC_NOTIFY))  in __trace_note_message()
168 	buf = this_cpu_ptr(bt->msg_data);  in __trace_note_message()
176 	trace_note(bt, current->pid, BLK_TN_MESSAGE, buf, n,  in __trace_note_message()
179 	trace_note(bt, current->pid, BLK_TN_MESSAGE, buf, n, 0);  in __trace_note_message()
185 static int act_log_check(struct blk_trace *bt, u32 what, sector_t sector,  in act_log_check()  argument
188 	if (((bt->act_mask << BLK_TC_SHIFT) & what) == 0)  in act_log_check()
190 	if (sector && (sector < bt->start_lba || sector > bt->end_lba))  in act_log_check()
192 	if (bt->pid && pid != bt->pid)  in act_log_check()
215 static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes,  in __blk_add_trace()  argument
231 	if (unlikely(bt->trace_state != Blktrace_running && !blk_tracer))  in __blk_add_trace()
248 	if (act_log_check(bt, what, sector, pid))  in __blk_add_trace()
275 	t = relay_reserve(bt->rchan, sizeof(*t) + pdu_len + cgid_len);  in __blk_add_trace()
277 		sequence = per_cpu_ptr(bt->sequence, cpu);  in __blk_add_trace()
295 		t->device = bt->dev;  in __blk_add_trace()
313 static void blk_trace_free(struct blk_trace *bt)  in blk_trace_free()  argument
315 	relay_close(bt->rchan);  in blk_trace_free()
316 	debugfs_remove(bt->dir);  in blk_trace_free()
317 	free_percpu(bt->sequence);  in blk_trace_free()
318 	free_percpu(bt->msg_data);  in blk_trace_free()
319 	kfree(bt);  in blk_trace_free()
338 static void blk_trace_cleanup(struct blk_trace *bt)  in blk_trace_cleanup()  argument
341 	blk_trace_free(bt);  in blk_trace_cleanup()
347 	struct blk_trace *bt;  in __blk_trace_remove()  local
349 	bt = rcu_replace_pointer(q->blk_trace, NULL,  in __blk_trace_remove()
351 	if (!bt)  in __blk_trace_remove()
354 	if (bt->trace_state != Blktrace_running)  in __blk_trace_remove()
355 		blk_trace_cleanup(bt);  in __blk_trace_remove()
375 	struct blk_trace *bt = filp->private_data;  in blk_dropped_read()  local
378 	snprintf(buf, sizeof(buf), "%u\n", atomic_read(&bt->dropped));  in blk_dropped_read()
394 	struct blk_trace *bt;  in blk_msg_write()  local
403 	bt = filp->private_data;  in blk_msg_write()
404 	__trace_note_message(bt, NULL, "%s", msg);  in blk_msg_write()
424 	struct blk_trace *bt;  in blk_subbuf_start_callback()  local
429 	bt = buf->chan->private_data;  in blk_subbuf_start_callback()
430 	atomic_inc(&bt->dropped);  in blk_subbuf_start_callback()
457 static void blk_trace_setup_lba(struct blk_trace *bt,  in blk_trace_setup_lba()  argument
461 		bt->start_lba = bdev->bd_start_sect;  in blk_trace_setup_lba()
462 		bt->end_lba = bdev->bd_start_sect + bdev_nr_sectors(bdev);  in blk_trace_setup_lba()
464 		bt->start_lba = 0;  in blk_trace_setup_lba()
465 		bt->end_lba = -1ULL;  in blk_trace_setup_lba()
476 	struct blk_trace *bt = NULL;  in do_blk_trace_setup()  local
505 	bt = kzalloc(sizeof(*bt), GFP_KERNEL);  in do_blk_trace_setup()
506 	if (!bt)  in do_blk_trace_setup()
510 	bt->sequence = alloc_percpu(unsigned long);  in do_blk_trace_setup()
511 	if (!bt->sequence)  in do_blk_trace_setup()
514 	bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char));  in do_blk_trace_setup()
515 	if (!bt->msg_data)  in do_blk_trace_setup()
527 		bt->dir = dir = debugfs_create_dir(buts->name, blk_debugfs_root);  in do_blk_trace_setup()
541 	bt->dev = dev;  in do_blk_trace_setup()
542 	atomic_set(&bt->dropped, 0);  in do_blk_trace_setup()
543 	INIT_LIST_HEAD(&bt->running_list);  in do_blk_trace_setup()
546 	debugfs_create_file("dropped", 0444, dir, bt, &blk_dropped_fops);  in do_blk_trace_setup()
547 	debugfs_create_file("msg", 0222, dir, bt, &blk_msg_fops);  in do_blk_trace_setup()
549 	bt->rchan = relay_open("trace", dir, buts->buf_size,  in do_blk_trace_setup()
550 				buts->buf_nr, &blk_relay_callbacks, bt);  in do_blk_trace_setup()
551 	if (!bt->rchan)  in do_blk_trace_setup()
554 	bt->act_mask = buts->act_mask;  in do_blk_trace_setup()
555 	if (!bt->act_mask)  in do_blk_trace_setup()
556 		bt->act_mask = (u16) -1;  in do_blk_trace_setup()
558 	blk_trace_setup_lba(bt, bdev);  in do_blk_trace_setup()
562 		bt->start_lba = buts->start_lba;  in do_blk_trace_setup()
564 		bt->end_lba = buts->end_lba;  in do_blk_trace_setup()
566 	bt->pid = buts->pid;  in do_blk_trace_setup()
567 	bt->trace_state = Blktrace_setup;  in do_blk_trace_setup()
569 	rcu_assign_pointer(q->blk_trace, bt);  in do_blk_trace_setup()
575 		blk_trace_free(bt);  in do_blk_trace_setup()
651 	struct blk_trace *bt;  in __blk_trace_startstop()  local
653 	bt = rcu_dereference_protected(q->blk_trace,  in __blk_trace_startstop()
655 	if (bt == NULL)  in __blk_trace_startstop()
664 		if (bt->trace_state == Blktrace_setup ||  in __blk_trace_startstop()
665 		    bt->trace_state == Blktrace_stopped) {  in __blk_trace_startstop()
668 			bt->trace_state = Blktrace_running;  in __blk_trace_startstop()
670 			list_add(&bt->running_list, &running_trace_list);  in __blk_trace_startstop()
673 			trace_note_time(bt);  in __blk_trace_startstop()
677 		if (bt->trace_state == Blktrace_running) {  in __blk_trace_startstop()
678 			bt->trace_state = Blktrace_stopped;  in __blk_trace_startstop()
680 			list_del_init(&bt->running_list);  in __blk_trace_startstop()
682 			relay_flush(bt->rchan);  in __blk_trace_startstop()
776 	struct blk_trace *bt;  in blk_trace_bio_get_cgid()  local
778 	/* We don't use the 'bt' value here except as an optimization... */  in blk_trace_bio_get_cgid()
779 	bt = rcu_dereference_protected(q->blk_trace, 1);  in blk_trace_bio_get_cgid()
780 	if (!bt || !(blk_tracer_flags.val & TRACE_BLK_OPT_CGROUP))  in blk_trace_bio_get_cgid()
822 	struct blk_trace *bt;  in blk_add_trace_rq()  local
825 	bt = rcu_dereference(rq->q->blk_trace);  in blk_add_trace_rq()
826 	if (likely(!bt)) {  in blk_add_trace_rq()
836 	__blk_add_trace(bt, blk_rq_trace_sector(rq), nr_bytes, req_op(rq),  in blk_add_trace_rq()
886 	struct blk_trace *bt;  in blk_add_trace_bio()  local
889 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_bio()
890 	if (likely(!bt)) {  in blk_add_trace_bio()
895 	__blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size,  in blk_add_trace_bio()
937 	struct blk_trace *bt;  in blk_add_trace_plug()  local
940 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_plug()
941 	if (bt)  in blk_add_trace_plug()
942 		__blk_add_trace(bt, 0, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL, 0);  in blk_add_trace_plug()
949 	struct blk_trace *bt;  in blk_add_trace_unplug()  local
952 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_unplug()
953 	if (bt) {  in blk_add_trace_unplug()
962 		__blk_add_trace(bt, 0, 0, 0, 0, what, 0, sizeof(rpdu), &rpdu, 0);  in blk_add_trace_unplug()
970 	struct blk_trace *bt;  in blk_add_trace_split()  local
973 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_split()
974 	if (bt) {  in blk_add_trace_split()
977 		__blk_add_trace(bt, bio->bi_iter.bi_sector,  in blk_add_trace_split()
1000 	struct blk_trace *bt;  in blk_add_trace_bio_remap()  local
1004 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_bio_remap()
1005 	if (likely(!bt)) {  in blk_add_trace_bio_remap()
1014 	__blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size,  in blk_add_trace_bio_remap()
1036 	struct blk_trace *bt;  in blk_add_trace_rq_remap()  local
1040 	bt = rcu_dereference(rq->q->blk_trace);  in blk_add_trace_rq_remap()
1041 	if (likely(!bt)) {  in blk_add_trace_rq_remap()
1050 	__blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq),  in blk_add_trace_rq_remap()
1068 	struct blk_trace *bt;  in blk_add_driver_data()  local
1071 	bt = rcu_dereference(rq->q->blk_trace);  in blk_add_driver_data()
1072 	if (likely(!bt)) {  in blk_add_driver_data()
1077 	__blk_add_trace(bt, blk_rq_trace_sector(rq), blk_rq_bytes(rq), 0, 0,  in blk_add_driver_data()
1601 	struct blk_trace *bt;  in blk_trace_remove_queue()  local
1603 	bt = rcu_replace_pointer(q->blk_trace, NULL,  in blk_trace_remove_queue()
1605 	if (bt == NULL)  in blk_trace_remove_queue()
1608 	if (bt->trace_state == Blktrace_running) {  in blk_trace_remove_queue()
1609 		bt->trace_state = Blktrace_stopped;  in blk_trace_remove_queue()
1611 		list_del_init(&bt->running_list);  in blk_trace_remove_queue()
1613 		relay_flush(bt->rchan);  in blk_trace_remove_queue()
1618 	blk_trace_free(bt);  in blk_trace_remove_queue()
1628 	struct blk_trace *bt = NULL;  in blk_trace_setup_queue()  local
1631 	bt = kzalloc(sizeof(*bt), GFP_KERNEL);  in blk_trace_setup_queue()
1632 	if (!bt)  in blk_trace_setup_queue()
1635 	bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char));  in blk_trace_setup_queue()
1636 	if (!bt->msg_data)  in blk_trace_setup_queue()
1639 	bt->dev = bdev->bd_dev;  in blk_trace_setup_queue()
1640 	bt->act_mask = (u16)-1;  in blk_trace_setup_queue()
1642 	blk_trace_setup_lba(bt, bdev);  in blk_trace_setup_queue()
1644 	rcu_assign_pointer(q->blk_trace, bt);  in blk_trace_setup_queue()
1649 	blk_trace_free(bt);  in blk_trace_setup_queue()
1767 	struct blk_trace *bt;  in sysfs_blk_trace_attr_show()  local
1772 	bt = rcu_dereference_protected(q->blk_trace,  in sysfs_blk_trace_attr_show()
1775 		ret = sprintf(buf, "%u\n", !!bt);  in sysfs_blk_trace_attr_show()
1779 	if (bt == NULL)  in sysfs_blk_trace_attr_show()
1782 		ret = blk_trace_mask2str(buf, bt->act_mask);  in sysfs_blk_trace_attr_show()
1784 		ret = sprintf(buf, "%u\n", bt->pid);  in sysfs_blk_trace_attr_show()
1786 		ret = sprintf(buf, "%llu\n", bt->start_lba);  in sysfs_blk_trace_attr_show()
1788 		ret = sprintf(buf, "%llu\n", bt->end_lba);  in sysfs_blk_trace_attr_show()
1801 	struct blk_trace *bt;  in sysfs_blk_trace_attr_store()  local
1823 	bt = rcu_dereference_protected(q->blk_trace,  in sysfs_blk_trace_attr_store()
1826 		if (!!value == !!bt) {  in sysfs_blk_trace_attr_store()
1838 	if (bt == NULL) {  in sysfs_blk_trace_attr_store()
1840 		bt = rcu_dereference_protected(q->blk_trace,  in sysfs_blk_trace_attr_store()
1846 			bt->act_mask = value;  in sysfs_blk_trace_attr_store()
1848 			bt->pid = value;  in sysfs_blk_trace_attr_store()
1850 			bt->start_lba = value;  in sysfs_blk_trace_attr_store()
1852 			bt->end_lba = value;  in sysfs_blk_trace_attr_store()