157 drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
169 static void store_vblank(struct drm_device *dev, unsigned int pipe,  in store_vblank()  argument
173 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in store_vblank()
185 static u32 drm_max_vblank_count(struct drm_device *dev, unsigned int pipe)  in drm_max_vblank_count()  argument
187 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_max_vblank_count()
196 static u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe)  in drm_vblank_no_hw_counter()  argument
198 	drm_WARN_ON_ONCE(dev, drm_max_vblank_count(dev, pipe) != 0);  in drm_vblank_no_hw_counter()
202 static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe)  in __get_vblank_counter()  argument
205 		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);  in __get_vblank_counter()
213 		return dev->driver->get_vblank_counter(dev, pipe);  in __get_vblank_counter()
216 	return drm_vblank_no_hw_counter(dev, pipe);  in __get_vblank_counter()
228 static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe)  in drm_reset_vblank_timestamp()  argument
242 		cur_vblank = __get_vblank_counter(dev, pipe);  in drm_reset_vblank_timestamp()
243 		rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);  in drm_reset_vblank_timestamp()
244 	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);  in drm_reset_vblank_timestamp()
258 	store_vblank(dev, pipe, 1, t_vblank, cur_vblank);  in drm_reset_vblank_timestamp()
275 static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,  in drm_update_vblank_count()  argument
278 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_update_vblank_count()
284 	u32 max_vblank_count = drm_max_vblank_count(dev, pipe);  in drm_update_vblank_count()
299 		cur_vblank = __get_vblank_counter(dev, pipe);  in drm_update_vblank_count()
300 		rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, in_vblank_irq);  in drm_update_vblank_count()
301 	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);  in drm_update_vblank_count()
317 			    pipe, (long long)diff_ns, framedur_ns);  in drm_update_vblank_count()
323 				    pipe);  in drm_update_vblank_count()
341 			    " due to pre-modeset.\n", pipe, diff);  in drm_update_vblank_count()
347 		    pipe, (unsigned long long)atomic64_read(&vblank->count),  in drm_update_vblank_count()
364 	store_vblank(dev, pipe, diff, t_vblank, cur_vblank);  in drm_update_vblank_count()
367 u64 drm_vblank_count(struct drm_device *dev, unsigned int pipe)  in drm_vblank_count()  argument
369 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_vblank_count()
372 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))  in drm_vblank_count()
403 	unsigned int pipe = drm_crtc_index(crtc);  in drm_crtc_accurate_vblank_count()  local
413 	drm_update_vblank_count(dev, pipe, false);  in drm_crtc_accurate_vblank_count()
414 	vblank = drm_vblank_count(dev, pipe);  in drm_crtc_accurate_vblank_count()
422 static void __disable_vblank(struct drm_device *dev, unsigned int pipe)  in __disable_vblank()  argument
425 		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);  in __disable_vblank()
433 		dev->driver->disable_vblank(dev, pipe);  in __disable_vblank()
443 void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe)  in drm_vblank_disable_and_save()  argument
445 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_vblank_disable_and_save()
471 	drm_update_vblank_count(dev, pipe, false);  in drm_vblank_disable_and_save()
472 	__disable_vblank(dev, pipe);  in drm_vblank_disable_and_save()
483 	unsigned int pipe = vblank->pipe;  in vblank_disable_fn()  local
488 		drm_dbg_core(dev, "disabling vblank on crtc %u\n", pipe);  in vblank_disable_fn()
489 		drm_vblank_disable_and_save(dev, pipe);  in vblank_disable_fn()
535 		vblank->pipe = i;  in drm_vblank_init()
606 	unsigned int pipe = drm_crtc_index(crtc);  in drm_calc_timestamping_constants()  local
607 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_calc_timestamping_constants()
614 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))  in drm_calc_timestamping_constants()
691 	unsigned int pipe = crtc->index;  in drm_crtc_vblank_helper_get_vblank_timestamp_internal()  local
692 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_crtc_vblank_helper_get_vblank_timestamp_internal()
700 	if (pipe >= dev->num_crtcs) {  in drm_crtc_vblank_helper_get_vblank_timestamp_internal()
701 		drm_err(dev, "Invalid crtc %u\n", pipe);  in drm_crtc_vblank_helper_get_vblank_timestamp_internal()
721 			     pipe);  in drm_crtc_vblank_helper_get_vblank_timestamp_internal()
747 				     pipe);  in drm_crtc_vblank_helper_get_vblank_timestamp_internal()
763 			     pipe, duration_ns / 1000, *max_error / 1000, i);  in drm_crtc_vblank_helper_get_vblank_timestamp_internal()
789 		    pipe, hpos, vpos,  in drm_crtc_vblank_helper_get_vblank_timestamp_internal()
862 drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,  in drm_get_last_vbltimestamp()  argument
865 	struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);  in drm_get_last_vbltimestamp()
873 		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);  in drm_get_last_vbltimestamp()
929 static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,  in drm_vblank_count_and_time()  argument
932 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_vblank_count_and_time()
936 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) {  in drm_vblank_count_and_time()
1002 	trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq);  in send_vblank_event()
1048 	unsigned int pipe = drm_crtc_index(crtc);  in drm_crtc_arm_vblank_event()  local
1052 	e->pipe = pipe;  in drm_crtc_arm_vblank_event()
1074 	unsigned int pipe = drm_crtc_index(crtc);  in drm_crtc_send_vblank_event()  local
1078 		seq = drm_vblank_count_and_time(dev, pipe, &now);  in drm_crtc_send_vblank_event()
1084 	e->pipe = pipe;  in drm_crtc_send_vblank_event()
1089 static int __enable_vblank(struct drm_device *dev, unsigned int pipe)  in __enable_vblank()  argument
1092 		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);  in __enable_vblank()
1100 		return dev->driver->enable_vblank(dev, pipe);  in __enable_vblank()
1106 static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)  in drm_vblank_enable()  argument
1108 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_vblank_enable()
1123 		ret = __enable_vblank(dev, pipe);  in drm_vblank_enable()
1125 			     pipe, ret);  in drm_vblank_enable()
1129 			drm_update_vblank_count(dev, pipe, 0);  in drm_vblank_enable()
1144 int drm_vblank_get(struct drm_device *dev, unsigned int pipe)  in drm_vblank_get()  argument
1146 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_vblank_get()
1153 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))  in drm_vblank_get()
1159 		ret = drm_vblank_enable(dev, pipe);  in drm_vblank_get()
1187 void drm_vblank_put(struct drm_device *dev, unsigned int pipe)  in drm_vblank_put()  argument
1189 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_vblank_put()
1191 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))  in drm_vblank_put()
1233 void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)  in drm_wait_one_vblank()  argument
1235 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_wait_one_vblank()
1239 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))  in drm_wait_one_vblank()
1242 	ret = drm_vblank_get(dev, pipe);  in drm_wait_one_vblank()
1244 		     pipe, ret))  in drm_wait_one_vblank()
1247 	last = drm_vblank_count(dev, pipe);  in drm_wait_one_vblank()
1250 				 last != drm_vblank_count(dev, pipe),  in drm_wait_one_vblank()
1253 	drm_WARN(dev, ret == 0, "vblank wait timed out on crtc %i\n", pipe);  in drm_wait_one_vblank()
1255 	drm_vblank_put(dev, pipe);  in drm_wait_one_vblank()
1287 	unsigned int pipe = drm_crtc_index(crtc);  in drm_crtc_vblank_off()  local
1288 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_crtc_vblank_off()
1293 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))  in drm_crtc_vblank_off()
1304 		    pipe, vblank->enabled, vblank->inmodeset);  in drm_crtc_vblank_off()
1309 		drm_vblank_disable_and_save(dev, pipe);  in drm_crtc_vblank_off()
1324 	seq = drm_vblank_count_and_time(dev, pipe, &now);  in drm_crtc_vblank_off()
1327 		if (e->pipe != pipe)  in drm_crtc_vblank_off()
1333 		drm_vblank_put(dev, pipe);  in drm_crtc_vblank_off()
1366 	unsigned int pipe = drm_crtc_index(crtc);  in drm_crtc_vblank_reset()  local
1367 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_crtc_vblank_reset()
1406 	unsigned int pipe = drm_crtc_index(crtc);  in drm_crtc_set_max_vblank_count()  local
1407 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_crtc_set_max_vblank_count()
1429 	unsigned int pipe = drm_crtc_index(crtc);  in drm_crtc_vblank_on()  local
1430 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_crtc_vblank_on()
1432 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))  in drm_crtc_vblank_on()
1437 		    pipe, vblank->enabled, vblank->inmodeset);  in drm_crtc_vblank_on()
1445 	drm_reset_vblank_timestamp(dev, pipe);  in drm_crtc_vblank_on()
1452 		drm_WARN_ON(dev, drm_vblank_enable(dev, pipe));  in drm_crtc_vblank_on()
1470 void drm_vblank_restore(struct drm_device *dev, unsigned int pipe)  in drm_vblank_restore()  argument
1479 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))  in drm_vblank_restore()
1485 	vblank = &dev->vblank[pipe];  in drm_vblank_restore()
1492 		cur_vblank = __get_vblank_counter(dev, pipe);  in drm_vblank_restore()
1493 		drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);  in drm_vblank_restore()
1494 	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);  in drm_vblank_restore()
1504 	store_vblank(dev, pipe, diff, t_vblank, cur_vblank);  in drm_vblank_restore()
1525 					  unsigned int pipe)  in drm_legacy_vblank_pre_modeset()  argument
1527 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_legacy_vblank_pre_modeset()
1533 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))  in drm_legacy_vblank_pre_modeset()
1545 		if (drm_vblank_get(dev, pipe) == 0)  in drm_legacy_vblank_pre_modeset()
1551 					   unsigned int pipe)  in drm_legacy_vblank_post_modeset()  argument
1553 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_legacy_vblank_post_modeset()
1559 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))  in drm_legacy_vblank_post_modeset()
1564 		drm_reset_vblank_timestamp(dev, pipe);  in drm_legacy_vblank_post_modeset()
1568 			drm_vblank_put(dev, pipe);  in drm_legacy_vblank_post_modeset()
1578 	unsigned int pipe;  in drm_legacy_modeset_ctl_ioctl()  local
1588 	pipe = modeset->crtc;  in drm_legacy_modeset_ctl_ioctl()
1589 	if (pipe >= dev->num_crtcs)  in drm_legacy_modeset_ctl_ioctl()
1594 		drm_legacy_vblank_pre_modeset(dev, pipe);  in drm_legacy_modeset_ctl_ioctl()
1597 		drm_legacy_vblank_post_modeset(dev, pipe);  in drm_legacy_modeset_ctl_ioctl()
1606 static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,  in drm_queue_vblank_event()  argument
1611 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_queue_vblank_event()
1623 	e->pipe = pipe;  in drm_queue_vblank_event()
1629 		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);  in drm_queue_vblank_event()
1654 	seq = drm_vblank_count_and_time(dev, pipe, &now);  in drm_queue_vblank_event()
1657 		     req_seq, seq, pipe);  in drm_queue_vblank_event()
1659 	trace_drm_vblank_event_queued(file_priv, pipe, req_seq);  in drm_queue_vblank_event()
1663 		drm_vblank_put(dev, pipe);  in drm_queue_vblank_event()
1680 	drm_vblank_put(dev, pipe);  in drm_queue_vblank_event()
1711 static void drm_wait_vblank_reply(struct drm_device *dev, unsigned int pipe,  in drm_wait_vblank_reply()  argument
1722 	reply->sequence = drm_vblank_count_and_time(dev, pipe, &now);  in drm_wait_vblank_reply()
1737 	unsigned int flags, pipe, high_pipe;  in drm_wait_vblank_ioctl()  local
1765 		pipe = 0;  in drm_wait_vblank_ioctl()
1772 			pipe++;  in drm_wait_vblank_ioctl()
1775 		pipe = pipe_index;  in drm_wait_vblank_ioctl()
1778 	if (pipe >= dev->num_crtcs)  in drm_wait_vblank_ioctl()
1781 	vblank = &dev->vblank[pipe];  in drm_wait_vblank_ioctl()
1789 		drm_wait_vblank_reply(dev, pipe, &vblwait->reply);  in drm_wait_vblank_ioctl()
1793 	ret = drm_vblank_get(dev, pipe);  in drm_wait_vblank_ioctl()
1797 			     pipe, ret);  in drm_wait_vblank_ioctl()
1800 	seq = drm_vblank_count(dev, pipe);  in drm_wait_vblank_ioctl()
1827 		return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv);  in drm_wait_vblank_ioctl()
1834 			     req_seq, pipe);  in drm_wait_vblank_ioctl()
1836 			drm_vblank_passed(drm_vblank_count(dev, pipe), req_seq) ||  in drm_wait_vblank_ioctl()
1856 		drm_wait_vblank_reply(dev, pipe, &vblwait->reply);  in drm_wait_vblank_ioctl()
1859 			     pipe, vblwait->reply.sequence);  in drm_wait_vblank_ioctl()
1862 			     pipe);  in drm_wait_vblank_ioctl()
1866 	drm_vblank_put(dev, pipe);  in drm_wait_vblank_ioctl()
1870 static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe)  in drm_handle_vblank_events()  argument
1872 	struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);  in drm_handle_vblank_events()
1880 	seq = drm_vblank_count_and_time(dev, pipe, &now);  in drm_handle_vblank_events()
1883 		if (e->pipe != pipe)  in drm_handle_vblank_events()
1892 		drm_vblank_put(dev, pipe);  in drm_handle_vblank_events()
1899 	trace_drm_vblank_event(pipe, seq, now, high_prec);  in drm_handle_vblank_events()
1912 bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe)  in drm_handle_vblank()  argument
1914 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_handle_vblank()
1921 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))  in drm_handle_vblank()
1939 	drm_update_vblank_count(dev, pipe, true);  in drm_handle_vblank()
1954 	drm_handle_vblank_events(dev, pipe);  in drm_handle_vblank()
2005 	int pipe;  in drm_crtc_get_sequence_ioctl()  local
2021 	pipe = drm_crtc_index(crtc);  in drm_crtc_get_sequence_ioctl()
2023 	vblank = &dev->vblank[pipe];  in drm_crtc_get_sequence_ioctl()
2031 				     pipe, ret);  in drm_crtc_get_sequence_ioctl()
2041 	get_seq->sequence = drm_vblank_count_and_time(dev, pipe, &now);  in drm_crtc_get_sequence_ioctl()
2061 	int pipe;  in drm_crtc_queue_sequence_ioctl()  local
2086 	pipe = drm_crtc_index(crtc);  in drm_crtc_queue_sequence_ioctl()
2088 	vblank = &dev->vblank[pipe];  in drm_crtc_queue_sequence_ioctl()
2098 			     pipe, ret);  in drm_crtc_queue_sequence_ioctl()
2102 	seq = drm_vblank_count_and_time(dev, pipe, &now);  in drm_crtc_queue_sequence_ioctl()
2111 	e->pipe = pipe;  in drm_crtc_queue_sequence_ioctl()