307 	struct intel_engine_cs *engine;  in gen6_reset_engines()  local
316 		for_each_engine_masked(engine, gt, engine_mask, tmp) {  in gen6_reset_engines()
317 			GEM_BUG_ON(engine->id >= ARRAY_SIZE(hw_engine_mask));  in gen6_reset_engines()
318 			hw_mask |= hw_engine_mask[engine->id];  in gen6_reset_engines()
325 static struct intel_engine_cs *find_sfc_paired_vecs_engine(struct intel_engine_cs *engine)  in find_sfc_paired_vecs_engine()  argument
329 	GEM_BUG_ON(engine->class != VIDEO_DECODE_CLASS);  in find_sfc_paired_vecs_engine()
331 	vecs_id = _VECS((engine->instance) / 2);  in find_sfc_paired_vecs_engine()
333 	return engine->gt->engine[vecs_id];  in find_sfc_paired_vecs_engine()
346 static void get_sfc_forced_lock_data(struct intel_engine_cs *engine,  in get_sfc_forced_lock_data()  argument
349 	switch (engine->class) {  in get_sfc_forced_lock_data()
351 		MISSING_CASE(engine->class);  in get_sfc_forced_lock_data()
354 		sfc_lock->lock_reg = GEN11_VCS_SFC_FORCED_LOCK(engine);  in get_sfc_forced_lock_data()
357 		sfc_lock->ack_reg = GEN11_VCS_SFC_LOCK_STATUS(engine);  in get_sfc_forced_lock_data()
360 		sfc_lock->usage_reg = GEN11_VCS_SFC_LOCK_STATUS(engine);  in get_sfc_forced_lock_data()
362 		sfc_lock->reset_bit = GEN11_VCS_SFC_RESET_BIT(engine->instance);  in get_sfc_forced_lock_data()
366 		sfc_lock->lock_reg = GEN11_VECS_SFC_FORCED_LOCK(engine);  in get_sfc_forced_lock_data()
369 		sfc_lock->ack_reg = GEN11_VECS_SFC_LOCK_ACK(engine);  in get_sfc_forced_lock_data()
372 		sfc_lock->usage_reg = GEN11_VECS_SFC_USAGE(engine);  in get_sfc_forced_lock_data()
374 		sfc_lock->reset_bit = GEN11_VECS_SFC_RESET_BIT(engine->instance);  in get_sfc_forced_lock_data()
380 static int gen11_lock_sfc(struct intel_engine_cs *engine,  in gen11_lock_sfc()  argument
384 	struct intel_uncore *uncore = engine->uncore;  in gen11_lock_sfc()
385 	u8 vdbox_sfc_access = engine->gt->info.vdbox_sfc_access;  in gen11_lock_sfc()
390 	switch (engine->class) {  in gen11_lock_sfc()
392 		if ((BIT(engine->instance) & vdbox_sfc_access) == 0)  in gen11_lock_sfc()
397 		get_sfc_forced_lock_data(engine, &sfc_lock);  in gen11_lock_sfc()
407 		if (engine->class != VIDEO_DECODE_CLASS ||  in gen11_lock_sfc()
408 		    GRAPHICS_VER(engine->i915) != 12)  in gen11_lock_sfc()
416 		 * forced lock on the VE engine that shares the same SFC.  in gen11_lock_sfc()
419 					   GEN12_HCP_SFC_LOCK_STATUS(engine)) &  in gen11_lock_sfc()
423 		paired_vecs = find_sfc_paired_vecs_engine(engine);  in gen11_lock_sfc()
428 		*unlock_mask |= engine->mask;  in gen11_lock_sfc()
432 	 * If the engine is using an SFC, tell the engine that a software reset  in gen11_lock_sfc()
433 	 * is going to happen. The engine will then try to force lock the SFC.  in gen11_lock_sfc()
434 	 * If SFC ends up being locked to the engine we want to reset, we have  in gen11_lock_sfc()
449 	 * We should reset both the engine and the SFC if:  in gen11_lock_sfc()
450 	 *  - We were locking the SFC to this engine and the lock succeeded  in gen11_lock_sfc()
452 	 *  - We were locking the SFC to a different engine (Wa_14010733141)  in gen11_lock_sfc()
455 	 * Otherwise we need only reset the engine by itself and we can  in gen11_lock_sfc()
464 		ENGINE_TRACE(engine, "Wait for SFC forced lock ack failed\n");  in gen11_lock_sfc()
472 static void gen11_unlock_sfc(struct intel_engine_cs *engine)  in gen11_unlock_sfc()  argument
474 	struct intel_uncore *uncore = engine->uncore;  in gen11_unlock_sfc()
475 	u8 vdbox_sfc_access = engine->gt->info.vdbox_sfc_access;  in gen11_unlock_sfc()
478 	if (engine->class != VIDEO_DECODE_CLASS &&  in gen11_unlock_sfc()
479 	    engine->class != VIDEO_ENHANCEMENT_CLASS)  in gen11_unlock_sfc()
482 	if (engine->class == VIDEO_DECODE_CLASS &&  in gen11_unlock_sfc()
483 	    (BIT(engine->instance) & vdbox_sfc_access) == 0)  in gen11_unlock_sfc()
486 	get_sfc_forced_lock_data(engine, &sfc_lock);  in gen11_unlock_sfc()
511 	struct intel_engine_cs *engine;  in gen11_reset_engines()  local
520 		for_each_engine_masked(engine, gt, engine_mask, tmp) {  in gen11_reset_engines()
521 			GEM_BUG_ON(engine->id >= ARRAY_SIZE(hw_engine_mask));  in gen11_reset_engines()
522 			reset_mask |= hw_engine_mask[engine->id];  in gen11_reset_engines()
523 			ret = gen11_lock_sfc(engine, &reset_mask, &unlock_mask);  in gen11_reset_engines()
538 	 * Due to Wa_14010733141, we may have locked an SFC to an engine that  in gen11_reset_engines()
543 	for_each_engine_masked(engine, gt, unlock_mask, tmp)  in gen11_reset_engines()
544 		gen11_unlock_sfc(engine);  in gen11_reset_engines()
549 static int gen8_engine_reset_prepare(struct intel_engine_cs *engine)  in gen8_engine_reset_prepare()  argument
551 	struct intel_uncore *uncore = engine->uncore;  in gen8_engine_reset_prepare()
552 	const i915_reg_t reg = RING_RESET_CTL(engine->mmio_base);  in gen8_engine_reset_prepare()
556 	if (I915_SELFTEST_ONLY(should_fail(&engine->reset_timeout, 1)))  in gen8_engine_reset_prepare()
582 		drm_err(&engine->i915->drm,  in gen8_engine_reset_prepare()
584 			engine->name, request,  in gen8_engine_reset_prepare()
590 static void gen8_engine_reset_cancel(struct intel_engine_cs *engine)  in gen8_engine_reset_cancel()  argument
592 	intel_uncore_write_fw(engine->uncore,  in gen8_engine_reset_cancel()
593 			      RING_RESET_CTL(engine->mmio_base),  in gen8_engine_reset_cancel()
601 	struct intel_engine_cs *engine;  in gen8_reset_engines()  local
606 	for_each_engine_masked(engine, gt, engine_mask, tmp) {  in gen8_reset_engines()
607 		ret = gen8_engine_reset_prepare(engine);  in gen8_reset_engines()
632 	for_each_engine_masked(engine, gt, engine_mask, tmp)  in gen8_reset_engines()
633 		gen8_engine_reset_cancel(engine);  in gen8_reset_engines()
733 static void reset_prepare_engine(struct intel_engine_cs *engine)  in reset_prepare_engine()  argument
736 	 * During the reset sequence, we must prevent the engine from  in reset_prepare_engine()
738 	 * the engine, if it does enter RC6 during the reset, the state  in reset_prepare_engine()
742 	intel_uncore_forcewake_get(engine->uncore, FORCEWAKE_ALL);  in reset_prepare_engine()
743 	if (engine->reset.prepare)  in reset_prepare_engine()
744 		engine->reset.prepare(engine);  in reset_prepare_engine()
780 	struct intel_engine_cs *engine;  in reset_prepare()  local
784 	for_each_engine(engine, gt, id) {  in reset_prepare()
785 		if (intel_engine_pm_get_if_awake(engine))  in reset_prepare()
786 			awake |= engine->mask;  in reset_prepare()
787 		reset_prepare_engine(engine);  in reset_prepare()
802 	struct intel_engine_cs *engine;  in gt_reset()  local
815 	for_each_engine(engine, gt, id)  in gt_reset()
816 		__intel_engine_reset(engine, stalled_mask & engine->mask);  in gt_reset()
826 static void reset_finish_engine(struct intel_engine_cs *engine)  in reset_finish_engine()  argument
828 	if (engine->reset.finish)  in reset_finish_engine()
829 		engine->reset.finish(engine);  in reset_finish_engine()
830 	intel_uncore_forcewake_put(engine->uncore, FORCEWAKE_ALL);  in reset_finish_engine()
832 	intel_engine_signal_breadcrumbs(engine);  in reset_finish_engine()
837 	struct intel_engine_cs *engine;  in reset_finish()  local
840 	for_each_engine(engine, gt, id) {  in reset_finish()
841 		reset_finish_engine(engine);  in reset_finish()
842 		if (awake & engine->mask)  in reset_finish()
843 			intel_engine_pm_put(engine);  in reset_finish()
856 		intel_engine_signal_breadcrumbs(request->engine);  in nop_submit_request()
864 	struct intel_engine_cs *engine;  in __intel_gt_set_wedged()  local
884 	for_each_engine(engine, gt, id)  in __intel_gt_set_wedged()
885 		engine->submit_request = nop_submit_request;  in __intel_gt_set_wedged()
897 	for_each_engine(engine, gt, id)  in __intel_gt_set_wedged()
898 		if (engine->reset.cancel)  in __intel_gt_set_wedged()
899 			engine->reset.cancel(engine);  in __intel_gt_set_wedged()
920 		struct intel_engine_cs *engine;  in intel_gt_set_wedged()  local
924 		for_each_engine(engine, gt, id) {  in intel_gt_set_wedged()
925 			if (intel_engine_is_idle(engine))  in intel_gt_set_wedged()
928 			intel_engine_dump(engine, &p, "%s\n", engine->name);  in intel_gt_set_wedged()
1007 	 * engine->submit_request() as we swap over. So unlike installing  in __intel_gt_unset_wedged()
1049 	struct intel_engine_cs *engine;  in resume()  local
1053 	for_each_engine(engine, gt, id) {  in resume()
1054 		ret = intel_engine_resume(engine);  in resume()
1176 static int intel_gt_reset_engine(struct intel_engine_cs *engine)  in intel_gt_reset_engine()  argument
1178 	return __intel_gt_reset(engine->gt, engine->mask);  in intel_gt_reset_engine()
1181 int __intel_engine_reset_bh(struct intel_engine_cs *engine, const char *msg)  in __intel_engine_reset_bh()  argument
1183 	struct intel_gt *gt = engine->gt;  in __intel_engine_reset_bh()
1186 	ENGINE_TRACE(engine, "flags=%lx\n", gt->reset.flags);  in __intel_engine_reset_bh()
1187 	GEM_BUG_ON(!test_bit(I915_RESET_ENGINE + engine->id, &gt->reset.flags));  in __intel_engine_reset_bh()
1189 	if (intel_engine_uses_guc(engine))  in __intel_engine_reset_bh()
1192 	if (!intel_engine_pm_get_if_awake(engine))  in __intel_engine_reset_bh()
1195 	reset_prepare_engine(engine);  in __intel_engine_reset_bh()
1198 		drm_notice(&engine->i915->drm,  in __intel_engine_reset_bh()
1199 			   "Resetting %s for %s\n", engine->name, msg);  in __intel_engine_reset_bh()
1200 	atomic_inc(&engine->i915->gpu_error.reset_engine_count[engine->uabi_class]);  in __intel_engine_reset_bh()
1202 	ret = intel_gt_reset_engine(engine);  in __intel_engine_reset_bh()
1205 		ENGINE_TRACE(engine, "Failed to reset %s, err: %d\n", engine->name, ret);  in __intel_engine_reset_bh()
1214 	__intel_engine_reset(engine, true);  in __intel_engine_reset_bh()
1217 	 * The engine and its registers (and workarounds in case of render)  in __intel_engine_reset_bh()
1221 	ret = intel_engine_resume(engine);  in __intel_engine_reset_bh()
1224 	intel_engine_cancel_stop_cs(engine);  in __intel_engine_reset_bh()
1225 	reset_finish_engine(engine);  in __intel_engine_reset_bh()
1226 	intel_engine_pm_put_async(engine);  in __intel_engine_reset_bh()
1231  * intel_engine_reset - reset GPU engine to recover from a hang
1232  * @engine: engine to reset
1235  * Reset a specific GPU engine. Useful if a hang is detected.
1240  *  - reset engine (which will force the engine to idle)
1241  *  - re-init/configure engine
1243 int intel_engine_reset(struct intel_engine_cs *engine, const char *msg)  in intel_engine_reset()  argument
1248 	err = __intel_engine_reset_bh(engine, msg);  in intel_engine_reset()
1303 	struct intel_engine_cs *engine;  in intel_gt_handle_error()  local
1336 	 * Try engine reset when available. We fall back to full reset if  in intel_gt_handle_error()
1342 		for_each_engine_masked(engine, gt, engine_mask, tmp) {  in intel_gt_handle_error()
1344 			if (test_and_set_bit(I915_RESET_ENGINE + engine->id,  in intel_gt_handle_error()
1348 			if (__intel_engine_reset_bh(engine, msg) == 0)  in intel_gt_handle_error()
1349 				engine_mask &= ~engine->mask;  in intel_gt_handle_error()
1351 			clear_and_wake_up_bit(I915_RESET_ENGINE + engine->id,  in intel_gt_handle_error()
1370 	/* Prevent any other reset-engine attempt. */  in intel_gt_handle_error()
1371 	for_each_engine(engine, gt, tmp) {  in intel_gt_handle_error()
1372 		while (test_and_set_bit(I915_RESET_ENGINE + engine->id,  in intel_gt_handle_error()
1375 				    I915_RESET_ENGINE + engine->id,  in intel_gt_handle_error()
1381 	for_each_engine(engine, gt, tmp)  in intel_gt_handle_error()
1382 		clear_bit_unlock(I915_RESET_ENGINE + engine->id,  in intel_gt_handle_error()