70 	delta = now - dev->power.accounting_timestamp;  in update_pm_runtime_accounting()
72 	dev->power.accounting_timestamp = now;  in update_pm_runtime_accounting()
74 	if (dev->power.disable_depth > 0)  in update_pm_runtime_accounting()
77 	if (dev->power.runtime_status == RPM_SUSPENDED)  in update_pm_runtime_accounting()
78 		dev->power.suspended_jiffies += delta;  in update_pm_runtime_accounting()
80 		dev->power.active_jiffies += delta;  in update_pm_runtime_accounting()
86 	dev->power.runtime_status = status;  in __update_runtime_status()
95 	if (dev->power.timer_expires > 0) {  in pm_runtime_deactivate_timer()
96 		del_timer(&dev->power.suspend_timer);  in pm_runtime_deactivate_timer()
97 		dev->power.timer_expires = 0;  in pm_runtime_deactivate_timer()
112 	dev->power.request = RPM_REQ_NONE;  in pm_runtime_cancel_pending()
134 	if (!dev->power.use_autosuspend)  in pm_runtime_autosuspend_expiration()
137 	autosuspend_delay = READ_ONCE(dev->power.autosuspend_delay);  in pm_runtime_autosuspend_expiration()
141 	last_busy = READ_ONCE(dev->power.last_busy);  in pm_runtime_autosuspend_expiration()
164 	return dev->power.memalloc_noio;  in dev_memalloc_noio()
204 		spin_lock_irq(&dev->power.lock);  in pm_runtime_set_memalloc_noio()
205 		enabled = dev->power.memalloc_noio;  in pm_runtime_set_memalloc_noio()
206 		dev->power.memalloc_noio = enable;  in pm_runtime_set_memalloc_noio()
207 		spin_unlock_irq(&dev->power.lock);  in pm_runtime_set_memalloc_noio()
240 	if (dev->power.runtime_error)  in rpm_check_suspend_allowed()
242 	else if (dev->power.disable_depth > 0)  in rpm_check_suspend_allowed()
244 	else if (atomic_read(&dev->power.usage_count) > 0)  in rpm_check_suspend_allowed()
246 	else if (!dev->power.ignore_children &&  in rpm_check_suspend_allowed()
247 			atomic_read(&dev->power.child_count))  in rpm_check_suspend_allowed()
251 	else if ((dev->power.deferred_resume  in rpm_check_suspend_allowed()
252 			&& dev->power.runtime_status == RPM_SUSPENDING)  in rpm_check_suspend_allowed()
253 	    || (dev->power.request_pending  in rpm_check_suspend_allowed()
254 			&& dev->power.request == RPM_REQ_RESUME))  in rpm_check_suspend_allowed()
258 	else if (dev->power.runtime_status == RPM_SUSPENDED)  in rpm_check_suspend_allowed()
307 	__releases(&dev->power.lock) __acquires(&dev->power.lock)  in __rpm_callback()
310 	bool use_links = dev->power.links_count > 0;  in __rpm_callback()
312 	if (dev->power.irq_safe) {  in __rpm_callback()
313 		spin_unlock(&dev->power.lock);  in __rpm_callback()
315 		spin_unlock_irq(&dev->power.lock);  in __rpm_callback()
324 		if (use_links && dev->power.runtime_status == RPM_RESUMING) {  in __rpm_callback()
337 	if (dev->power.irq_safe) {  in __rpm_callback()
338 		spin_lock(&dev->power.lock);  in __rpm_callback()
348 		    && ((dev->power.runtime_status == RPM_SUSPENDING && !retval)  in __rpm_callback()
349 		    || (dev->power.runtime_status == RPM_RESUMING && retval))) {  in __rpm_callback()
358 		spin_lock_irq(&dev->power.lock);  in __rpm_callback()
388 	else if (dev->power.runtime_status != RPM_ACTIVE)  in rpm_idle()
395 	else if (dev->power.request_pending &&  in rpm_idle()
396 	    dev->power.request > RPM_REQ_IDLE)  in rpm_idle()
400 	else if (dev->power.idle_notification)  in rpm_idle()
406 	dev->power.request = RPM_REQ_NONE;  in rpm_idle()
408 	if (dev->power.no_callbacks)  in rpm_idle()
413 		dev->power.request = RPM_REQ_IDLE;  in rpm_idle()
414 		if (!dev->power.request_pending) {  in rpm_idle()
415 			dev->power.request_pending = true;  in rpm_idle()
416 			queue_work(pm_wq, &dev->power.work);  in rpm_idle()
422 	dev->power.idle_notification = true;  in rpm_idle()
429 	dev->power.idle_notification = false;  in rpm_idle()
430 	wake_up_all(&dev->power.wait_queue);  in rpm_idle()
449 	if (dev->power.memalloc_noio) {  in rpm_callback()
468 	dev->power.runtime_error = retval;  in rpm_callback()
494 	__releases(&dev->power.lock) __acquires(&dev->power.lock)  in rpm_suspend()
509 	else if (dev->power.runtime_status == RPM_RESUMING &&  in rpm_suspend()
517 	    && dev->power.runtime_status != RPM_SUSPENDING) {  in rpm_suspend()
522 			dev->power.request = RPM_REQ_NONE;  in rpm_suspend()
531 			if (!(dev->power.timer_expires && time_before_eq(  in rpm_suspend()
532 			    dev->power.timer_expires, expires))) {  in rpm_suspend()
533 				dev->power.timer_expires = expires;  in rpm_suspend()
534 				mod_timer(&dev->power.suspend_timer, expires);  in rpm_suspend()
536 			dev->power.timer_autosuspends = 1;  in rpm_suspend()
544 	if (dev->power.runtime_status == RPM_SUSPENDING) {  in rpm_suspend()
552 		if (dev->power.irq_safe) {  in rpm_suspend()
553 			spin_unlock(&dev->power.lock);  in rpm_suspend()
557 			spin_lock(&dev->power.lock);  in rpm_suspend()
563 			prepare_to_wait(&dev->power.wait_queue, &wait,  in rpm_suspend()
565 			if (dev->power.runtime_status != RPM_SUSPENDING)  in rpm_suspend()
568 			spin_unlock_irq(&dev->power.lock);  in rpm_suspend()
572 			spin_lock_irq(&dev->power.lock);  in rpm_suspend()
574 		finish_wait(&dev->power.wait_queue, &wait);  in rpm_suspend()
578 	if (dev->power.no_callbacks)  in rpm_suspend()
583 		dev->power.request = (rpmflags & RPM_AUTO) ?  in rpm_suspend()
585 		if (!dev->power.request_pending) {  in rpm_suspend()
586 			dev->power.request_pending = true;  in rpm_suspend()
587 			queue_work(pm_wq, &dev->power.work);  in rpm_suspend()
607 		atomic_add_unless(&parent->power.child_count, -1, 0);  in rpm_suspend()
609 	wake_up_all(&dev->power.wait_queue);  in rpm_suspend()
611 	if (dev->power.deferred_resume) {  in rpm_suspend()
612 		dev->power.deferred_resume = false;  in rpm_suspend()
619 	if (parent && !parent->power.ignore_children && !dev->power.irq_safe) {  in rpm_suspend()
620 		spin_unlock(&dev->power.lock);  in rpm_suspend()
622 		spin_lock(&parent->power.lock);  in rpm_suspend()
624 		spin_unlock(&parent->power.lock);  in rpm_suspend()
626 		spin_lock(&dev->power.lock);  in rpm_suspend()
637 	dev->power.deferred_resume = false;  in rpm_suspend()
638 	wake_up_all(&dev->power.wait_queue);  in rpm_suspend()
641 		dev->power.runtime_error = 0;  in rpm_suspend()
676 	__releases(&dev->power.lock) __acquires(&dev->power.lock)  in rpm_resume()
685 	if (dev->power.runtime_error)  in rpm_resume()
687 	else if (dev->power.disable_depth == 1 && dev->power.is_suspended  in rpm_resume()
688 	    && dev->power.runtime_status == RPM_ACTIVE)  in rpm_resume()
690 	else if (dev->power.disable_depth > 0)  in rpm_resume()
701 	dev->power.request = RPM_REQ_NONE;  in rpm_resume()
702 	if (!dev->power.timer_autosuspends)  in rpm_resume()
705 	if (dev->power.runtime_status == RPM_ACTIVE) {  in rpm_resume()
710 	if (dev->power.runtime_status == RPM_RESUMING  in rpm_resume()
711 	    || dev->power.runtime_status == RPM_SUSPENDING) {  in rpm_resume()
715 			if (dev->power.runtime_status == RPM_SUSPENDING)  in rpm_resume()
716 				dev->power.deferred_resume = true;  in rpm_resume()
722 		if (dev->power.irq_safe) {  in rpm_resume()
723 			spin_unlock(&dev->power.lock);  in rpm_resume()
727 			spin_lock(&dev->power.lock);  in rpm_resume()
733 			prepare_to_wait(&dev->power.wait_queue, &wait,  in rpm_resume()
735 			if (dev->power.runtime_status != RPM_RESUMING  in rpm_resume()
736 			    && dev->power.runtime_status != RPM_SUSPENDING)  in rpm_resume()
739 			spin_unlock_irq(&dev->power.lock);  in rpm_resume()
743 			spin_lock_irq(&dev->power.lock);  in rpm_resume()
745 		finish_wait(&dev->power.wait_queue, &wait);  in rpm_resume()
754 	if (dev->power.no_callbacks && !parent && dev->parent) {  in rpm_resume()
755 		spin_lock_nested(&dev->parent->power.lock, SINGLE_DEPTH_NESTING);  in rpm_resume()
756 		if (dev->parent->power.disable_depth > 0  in rpm_resume()
757 		    || dev->parent->power.ignore_children  in rpm_resume()
758 		    || dev->parent->power.runtime_status == RPM_ACTIVE) {  in rpm_resume()
759 			atomic_inc(&dev->parent->power.child_count);  in rpm_resume()
760 			spin_unlock(&dev->parent->power.lock);  in rpm_resume()
764 		spin_unlock(&dev->parent->power.lock);  in rpm_resume()
769 		dev->power.request = RPM_REQ_RESUME;  in rpm_resume()
770 		if (!dev->power.request_pending) {  in rpm_resume()
771 			dev->power.request_pending = true;  in rpm_resume()
772 			queue_work(pm_wq, &dev->power.work);  in rpm_resume()
785 		if (dev->power.irq_safe)  in rpm_resume()
787 		spin_unlock(&dev->power.lock);  in rpm_resume()
791 		spin_lock(&parent->power.lock);  in rpm_resume()
796 		if (!parent->power.disable_depth  in rpm_resume()
797 		    && !parent->power.ignore_children) {  in rpm_resume()
799 			if (parent->power.runtime_status != RPM_ACTIVE)  in rpm_resume()
802 		spin_unlock(&parent->power.lock);  in rpm_resume()
804 		spin_lock(&dev->power.lock);  in rpm_resume()
811 	if (dev->power.no_callbacks)  in rpm_resume()
829 			atomic_inc(&parent->power.child_count);  in rpm_resume()
831 	wake_up_all(&dev->power.wait_queue);  in rpm_resume()
837 	if (parent && !dev->power.irq_safe) {  in rpm_resume()
838 		spin_unlock_irq(&dev->power.lock);  in rpm_resume()
842 		spin_lock_irq(&dev->power.lock);  in rpm_resume()
859 	struct device *dev = container_of(work, struct device, power.work);  in pm_runtime_work()
862 	spin_lock_irq(&dev->power.lock);  in pm_runtime_work()
864 	if (!dev->power.request_pending)  in pm_runtime_work()
867 	req = dev->power.request;  in pm_runtime_work()
868 	dev->power.request = RPM_REQ_NONE;  in pm_runtime_work()
869 	dev->power.request_pending = false;  in pm_runtime_work()
889 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_work()
900 	struct device *dev = from_timer(dev, t, power.suspend_timer);  in pm_suspend_timer_fn()
904 	spin_lock_irqsave(&dev->power.lock, flags);  in pm_suspend_timer_fn()
906 	expires = dev->power.timer_expires;  in pm_suspend_timer_fn()
909 		dev->power.timer_expires = 0;  in pm_suspend_timer_fn()
910 		rpm_suspend(dev, dev->power.timer_autosuspends ?  in pm_suspend_timer_fn()
914 	spin_unlock_irqrestore(&dev->power.lock, flags);  in pm_suspend_timer_fn()
927 	spin_lock_irqsave(&dev->power.lock, flags);  in pm_schedule_suspend()
941 	dev->power.timer_expires = jiffies + msecs_to_jiffies(delay);  in pm_schedule_suspend()
942 	dev->power.timer_expires += !dev->power.timer_expires;  in pm_schedule_suspend()
943 	dev->power.timer_autosuspends = 0;  in pm_schedule_suspend()
944 	mod_timer(&dev->power.suspend_timer, dev->power.timer_expires);  in pm_schedule_suspend()
947 	spin_unlock_irqrestore(&dev->power.lock, flags);  in pm_schedule_suspend()
971 		if (!atomic_dec_and_test(&dev->power.usage_count))  in __pm_runtime_idle()
975 	might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe);  in __pm_runtime_idle()
977 	spin_lock_irqsave(&dev->power.lock, flags);  in __pm_runtime_idle()
979 	spin_unlock_irqrestore(&dev->power.lock, flags);  in __pm_runtime_idle()
1003 		if (!atomic_dec_and_test(&dev->power.usage_count))  in __pm_runtime_suspend()
1007 	might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe);  in __pm_runtime_suspend()
1009 	spin_lock_irqsave(&dev->power.lock, flags);  in __pm_runtime_suspend()
1011 	spin_unlock_irqrestore(&dev->power.lock, flags);  in __pm_runtime_suspend()
1033 	might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe &&  in __pm_runtime_resume()
1034 			dev->power.runtime_status != RPM_ACTIVE);  in __pm_runtime_resume()
1037 		atomic_inc(&dev->power.usage_count);  in __pm_runtime_resume()
1039 	spin_lock_irqsave(&dev->power.lock, flags);  in __pm_runtime_resume()
1041 	spin_unlock_irqrestore(&dev->power.lock, flags);  in __pm_runtime_resume()
1062 	spin_lock_irqsave(&dev->power.lock, flags);  in pm_runtime_get_if_in_use()
1063 	retval = dev->power.disable_depth > 0 ? -EINVAL :  in pm_runtime_get_if_in_use()
1064 		dev->power.runtime_status == RPM_ACTIVE  in pm_runtime_get_if_in_use()
1065 			&& atomic_inc_not_zero(&dev->power.usage_count);  in pm_runtime_get_if_in_use()
1066 	spin_unlock_irqrestore(&dev->power.lock, flags);  in pm_runtime_get_if_in_use()
1098 	spin_lock_irqsave(&dev->power.lock, flags);  in __pm_runtime_set_status()
1100 	if (!dev->power.runtime_error && !dev->power.disable_depth) {  in __pm_runtime_set_status()
1105 	if (dev->power.runtime_status == status || !parent)  in __pm_runtime_set_status()
1109 		atomic_add_unless(&parent->power.child_count, -1, 0);  in __pm_runtime_set_status()
1110 		notify_parent = !parent->power.ignore_children;  in __pm_runtime_set_status()
1112 		spin_lock_nested(&parent->power.lock, SINGLE_DEPTH_NESTING);  in __pm_runtime_set_status()
1119 		if (!parent->power.disable_depth  in __pm_runtime_set_status()
1120 		    && !parent->power.ignore_children  in __pm_runtime_set_status()
1121 		    && parent->power.runtime_status != RPM_ACTIVE) {  in __pm_runtime_set_status()
1126 		} else if (dev->power.runtime_status == RPM_SUSPENDED) {  in __pm_runtime_set_status()
1127 			atomic_inc(&parent->power.child_count);  in __pm_runtime_set_status()
1130 		spin_unlock(&parent->power.lock);  in __pm_runtime_set_status()
1138 	dev->power.runtime_error = 0;  in __pm_runtime_set_status()
1140 	spin_unlock_irqrestore(&dev->power.lock, flags);  in __pm_runtime_set_status()
1162 	if (dev->power.request_pending) {  in __pm_runtime_barrier()
1163 		dev->power.request = RPM_REQ_NONE;  in __pm_runtime_barrier()
1164 		spin_unlock_irq(&dev->power.lock);  in __pm_runtime_barrier()
1166 		cancel_work_sync(&dev->power.work);  in __pm_runtime_barrier()
1168 		spin_lock_irq(&dev->power.lock);  in __pm_runtime_barrier()
1169 		dev->power.request_pending = false;  in __pm_runtime_barrier()
1172 	if (dev->power.runtime_status == RPM_SUSPENDING  in __pm_runtime_barrier()
1173 	    || dev->power.runtime_status == RPM_RESUMING  in __pm_runtime_barrier()
1174 	    || dev->power.idle_notification) {  in __pm_runtime_barrier()
1179 			prepare_to_wait(&dev->power.wait_queue, &wait,  in __pm_runtime_barrier()
1181 			if (dev->power.runtime_status != RPM_SUSPENDING  in __pm_runtime_barrier()
1182 			    && dev->power.runtime_status != RPM_RESUMING  in __pm_runtime_barrier()
1183 			    && !dev->power.idle_notification)  in __pm_runtime_barrier()
1185 			spin_unlock_irq(&dev->power.lock);  in __pm_runtime_barrier()
1189 			spin_lock_irq(&dev->power.lock);  in __pm_runtime_barrier()
1191 		finish_wait(&dev->power.wait_queue, &wait);  in __pm_runtime_barrier()
1214 	spin_lock_irq(&dev->power.lock);  in pm_runtime_barrier()
1216 	if (dev->power.request_pending  in pm_runtime_barrier()
1217 	    && dev->power.request == RPM_REQ_RESUME) {  in pm_runtime_barrier()
1224 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_barrier()
1247 	spin_lock_irq(&dev->power.lock);  in __pm_runtime_disable()
1249 	if (dev->power.disable_depth > 0) {  in __pm_runtime_disable()
1250 		dev->power.disable_depth++;  in __pm_runtime_disable()
1259 	if (check_resume && dev->power.request_pending  in __pm_runtime_disable()
1260 	    && dev->power.request == RPM_REQ_RESUME) {  in __pm_runtime_disable()
1272 	if (!dev->power.disable_depth++)  in __pm_runtime_disable()
1276 	spin_unlock_irq(&dev->power.lock);  in __pm_runtime_disable()
1288 	spin_lock_irqsave(&dev->power.lock, flags);  in pm_runtime_enable()
1290 	if (dev->power.disable_depth > 0)  in pm_runtime_enable()
1291 		dev->power.disable_depth--;  in pm_runtime_enable()
1295 	WARN(!dev->power.disable_depth &&  in pm_runtime_enable()
1296 	     dev->power.runtime_status == RPM_SUSPENDED &&  in pm_runtime_enable()
1297 	     !dev->power.ignore_children &&  in pm_runtime_enable()
1298 	     atomic_read(&dev->power.child_count) > 0,  in pm_runtime_enable()
1302 	spin_unlock_irqrestore(&dev->power.lock, flags);  in pm_runtime_enable()
1316 	spin_lock_irq(&dev->power.lock);  in pm_runtime_forbid()
1317 	if (!dev->power.runtime_auto)  in pm_runtime_forbid()
1320 	dev->power.runtime_auto = false;  in pm_runtime_forbid()
1321 	atomic_inc(&dev->power.usage_count);  in pm_runtime_forbid()
1325 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_forbid()
1337 	spin_lock_irq(&dev->power.lock);  in pm_runtime_allow()
1338 	if (dev->power.runtime_auto)  in pm_runtime_allow()
1341 	dev->power.runtime_auto = true;  in pm_runtime_allow()
1342 	if (atomic_dec_and_test(&dev->power.usage_count))  in pm_runtime_allow()
1346 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_allow()
1360 	spin_lock_irq(&dev->power.lock);  in pm_runtime_no_callbacks()
1361 	dev->power.no_callbacks = 1;  in pm_runtime_no_callbacks()
1362 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_no_callbacks()
1383 	spin_lock_irq(&dev->power.lock);  in pm_runtime_irq_safe()
1384 	dev->power.irq_safe = 1;  in pm_runtime_irq_safe()
1385 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_irq_safe()
1402 	int delay = dev->power.autosuspend_delay;  in update_autosuspend()
1405 	if (dev->power.use_autosuspend && delay < 0) {  in update_autosuspend()
1409 			atomic_inc(&dev->power.usage_count);  in update_autosuspend()
1419 			atomic_dec(&dev->power.usage_count);  in update_autosuspend()
1439 	spin_lock_irq(&dev->power.lock);  in pm_runtime_set_autosuspend_delay()
1440 	old_delay = dev->power.autosuspend_delay;  in pm_runtime_set_autosuspend_delay()
1441 	old_use = dev->power.use_autosuspend;  in pm_runtime_set_autosuspend_delay()
1442 	dev->power.autosuspend_delay = delay;  in pm_runtime_set_autosuspend_delay()
1444 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_set_autosuspend_delay()
1460 	spin_lock_irq(&dev->power.lock);  in __pm_runtime_use_autosuspend()
1461 	old_delay = dev->power.autosuspend_delay;  in __pm_runtime_use_autosuspend()
1462 	old_use = dev->power.use_autosuspend;  in __pm_runtime_use_autosuspend()
1463 	dev->power.use_autosuspend = use;  in __pm_runtime_use_autosuspend()
1465 	spin_unlock_irq(&dev->power.lock);  in __pm_runtime_use_autosuspend()
1475 	dev->power.runtime_status = RPM_SUSPENDED;  in pm_runtime_init()
1476 	dev->power.idle_notification = false;  in pm_runtime_init()
1478 	dev->power.disable_depth = 1;  in pm_runtime_init()
1479 	atomic_set(&dev->power.usage_count, 0);  in pm_runtime_init()
1481 	dev->power.runtime_error = 0;  in pm_runtime_init()
1483 	atomic_set(&dev->power.child_count, 0);  in pm_runtime_init()
1485 	dev->power.runtime_auto = true;  in pm_runtime_init()
1487 	dev->power.request_pending = false;  in pm_runtime_init()
1488 	dev->power.request = RPM_REQ_NONE;  in pm_runtime_init()
1489 	dev->power.deferred_resume = false;  in pm_runtime_init()
1490 	dev->power.accounting_timestamp = jiffies;  in pm_runtime_init()
1491 	INIT_WORK(&dev->power.work, pm_runtime_work);  in pm_runtime_init()
1493 	dev->power.timer_expires = 0;  in pm_runtime_init()
1494 	timer_setup(&dev->power.suspend_timer, pm_suspend_timer_fn, 0);  in pm_runtime_init()
1496 	init_waitqueue_head(&dev->power.wait_queue);  in pm_runtime_init()
1506 		if (dev->power.runtime_status == RPM_ACTIVE)  in pm_runtime_reinit()
1508 		if (dev->power.irq_safe) {  in pm_runtime_reinit()
1509 			spin_lock_irq(&dev->power.lock);  in pm_runtime_reinit()
1510 			dev->power.irq_safe = 0;  in pm_runtime_reinit()
1511 			spin_unlock_irq(&dev->power.lock);  in pm_runtime_reinit()
1603 	spin_lock_irq(&dev->power.lock);  in pm_runtime_new_link()
1604 	dev->power.links_count++;  in pm_runtime_new_link()
1605 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_new_link()
1612 	spin_lock_irq(&dev->power.lock);  in pm_runtime_drop_link()
1613 	WARN_ON(dev->power.links_count == 0);  in pm_runtime_drop_link()
1614 	dev->power.links_count--;  in pm_runtime_drop_link()
1615 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_drop_link()
1620 	return atomic_read(&dev->power.usage_count) <= 1 &&  in pm_runtime_need_not_resume()
1621 		(atomic_read(&dev->power.child_count) == 0 ||  in pm_runtime_need_not_resume()
1622 		 dev->power.ignore_children);  in pm_runtime_need_not_resume()