100 		       const struct i915_request *rq)  in hws_address()  argument
102 	return hws->node.start + offset_in_page(sizeof(u32)*rq->fence.context);  in hws_address()
106 			      struct i915_request *rq)  in emit_recurse_batch()  argument
110 		rq->gem_context->ppgtt ?  in emit_recurse_batch()
111 		&rq->gem_context->ppgtt->vm :  in emit_recurse_batch()
134 	err = i915_vma_move_to_active(vma, rq, 0);  in emit_recurse_batch()
143 	err = i915_vma_move_to_active(hws, rq, 0);  in emit_recurse_batch()
155 		*batch++ = lower_32_bits(hws_address(hws, rq));  in emit_recurse_batch()
156 		*batch++ = upper_32_bits(hws_address(hws, rq));  in emit_recurse_batch()
157 		*batch++ = rq->fence.seqno;  in emit_recurse_batch()
170 		*batch++ = lower_32_bits(hws_address(hws, rq));  in emit_recurse_batch()
171 		*batch++ = rq->fence.seqno;  in emit_recurse_batch()
183 		*batch++ = lower_32_bits(hws_address(hws, rq));  in emit_recurse_batch()
184 		*batch++ = rq->fence.seqno;  in emit_recurse_batch()
195 		*batch++ = lower_32_bits(hws_address(hws, rq));  in emit_recurse_batch()
196 		*batch++ = rq->fence.seqno;  in emit_recurse_batch()
213 	err = rq->engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, flags);  in emit_recurse_batch()
225 	struct i915_request *rq;  in hang_create_request()  local
250 	rq = i915_request_alloc(engine, h->ctx);  in hang_create_request()
251 	if (IS_ERR(rq))  in hang_create_request()
252 		return rq;  in hang_create_request()
254 	err = emit_recurse_batch(h, rq);  in hang_create_request()
256 		i915_request_add(rq);  in hang_create_request()
260 	return rq;  in hang_create_request()
263 static u32 hws_seqno(const struct hang *h, const struct i915_request *rq)  in hws_seqno()  argument
265 	return READ_ONCE(h->seqno[rq->fence.context % (PAGE_SIZE/sizeof(u32))]);  in hws_seqno()
284 static bool wait_until_running(struct hang *h, struct i915_request *rq)  in wait_until_running()  argument
286 	return !(wait_for_us(i915_seqno_passed(hws_seqno(h, rq),  in wait_until_running()
287 					       rq->fence.seqno),  in wait_until_running()
289 		 wait_for(i915_seqno_passed(hws_seqno(h, rq),  in wait_until_running()
290 					    rq->fence.seqno),  in wait_until_running()
297 	struct i915_request *rq;  in igt_hang_sanitycheck()  local
316 		rq = hang_create_request(&h, engine);  in igt_hang_sanitycheck()
317 		if (IS_ERR(rq)) {  in igt_hang_sanitycheck()
318 			err = PTR_ERR(rq);  in igt_hang_sanitycheck()
324 		i915_request_get(rq);  in igt_hang_sanitycheck()
329 		i915_request_add(rq);  in igt_hang_sanitycheck()
331 		timeout = i915_request_wait(rq,  in igt_hang_sanitycheck()
334 		i915_request_put(rq);  in igt_hang_sanitycheck()
464 				struct i915_request *rq;  in __igt_reset_engine()  local
467 				rq = hang_create_request(&h, engine);  in __igt_reset_engine()
468 				if (IS_ERR(rq)) {  in __igt_reset_engine()
469 					err = PTR_ERR(rq);  in __igt_reset_engine()
474 				i915_request_get(rq);  in __igt_reset_engine()
475 				i915_request_add(rq);  in __igt_reset_engine()
478 				if (!wait_until_running(&h, rq)) {  in __igt_reset_engine()
482 					       __func__, rq->fence.seqno, hws_seqno(&h, rq));  in __igt_reset_engine()
486 					i915_request_put(rq);  in __igt_reset_engine()
491 				GEM_BUG_ON(!rq->global_seqno);  in __igt_reset_engine()
492 				seqno = rq->global_seqno - 1;  in __igt_reset_engine()
493 				i915_request_put(rq);  in __igt_reset_engine()
574 static int active_request_put(struct i915_request *rq)  in active_request_put()  argument
578 	if (!rq)  in active_request_put()
581 	if (i915_request_wait(rq, 0, 5 * HZ) < 0) {  in active_request_put()
583 			  rq->engine->name,  in active_request_put()
584 			  rq->fence.context,  in active_request_put()
585 			  rq->fence.seqno,  in active_request_put()
586 			  i915_request_global_seqno(rq));  in active_request_put()
589 		i915_gem_set_wedged(rq->i915);  in active_request_put()
593 	i915_request_put(rq);  in active_request_put()
603 	struct i915_request *rq[8] = {};  in active_engine()  local
604 	struct i915_gem_context *ctx[ARRAY_SIZE(rq)];  in active_engine()
626 		unsigned int idx = count++ & (ARRAY_SIZE(rq) - 1);  in active_engine()
627 		struct i915_request *old = rq[idx];  in active_engine()
642 		rq[idx] = i915_request_get(new);  in active_engine()
653 	for (count = 0; count < ARRAY_SIZE(rq); count++) {  in active_engine()
654 		int err__ = active_request_put(rq[count]);  in active_engine()
741 			struct i915_request *rq = NULL;  in __igt_reset_engines()  local
745 				rq = hang_create_request(&h, engine);  in __igt_reset_engines()
746 				if (IS_ERR(rq)) {  in __igt_reset_engines()
747 					err = PTR_ERR(rq);  in __igt_reset_engines()
752 				i915_request_get(rq);  in __igt_reset_engines()
753 				i915_request_add(rq);  in __igt_reset_engines()
756 				if (!wait_until_running(&h, rq)) {  in __igt_reset_engines()
760 					       __func__, rq->fence.seqno, hws_seqno(&h, rq));  in __igt_reset_engines()
764 					i915_request_put(rq);  in __igt_reset_engines()
769 				GEM_BUG_ON(!rq->global_seqno);  in __igt_reset_engines()
770 				seqno = rq->global_seqno - 1;  in __igt_reset_engines()
782 			if (rq) {  in __igt_reset_engines()
783 				i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT);  in __igt_reset_engines()
784 				i915_request_put(rq);  in __igt_reset_engines()
909 static u32 fake_hangcheck(struct i915_request *rq, u32 mask)  in fake_hangcheck()  argument
911 	struct i915_gpu_error *error = &rq->i915->gpu_error;  in fake_hangcheck()
928 	struct i915_request *rq;  in igt_reset_wait()  local
946 	rq = hang_create_request(&h, i915->engine[RCS]);  in igt_reset_wait()
947 	if (IS_ERR(rq)) {  in igt_reset_wait()
948 		err = PTR_ERR(rq);  in igt_reset_wait()
952 	i915_request_get(rq);  in igt_reset_wait()
953 	i915_request_add(rq);  in igt_reset_wait()
955 	if (!wait_until_running(&h, rq)) {  in igt_reset_wait()
959 		       __func__, rq->fence.seqno, hws_seqno(&h, rq));  in igt_reset_wait()
960 		intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);  in igt_reset_wait()
968 	reset_count = fake_hangcheck(rq, ALL_ENGINES);  in igt_reset_wait()
970 	timeout = i915_request_wait(rq, I915_WAIT_LOCKED, 10);  in igt_reset_wait()
986 	i915_request_put(rq);  in igt_reset_wait()
1026 	struct i915_request *rq;  in __igt_reset_evict_vma()  local
1055 	rq = hang_create_request(&h, i915->engine[RCS]);  in __igt_reset_evict_vma()
1056 	if (IS_ERR(rq)) {  in __igt_reset_evict_vma()
1057 		err = PTR_ERR(rq);  in __igt_reset_evict_vma()
1066 	err = i915_vma_move_to_active(arg.vma, rq, EXEC_OBJECT_WRITE);  in __igt_reset_evict_vma()
1069 	i915_request_get(rq);  in __igt_reset_evict_vma()
1070 	i915_request_add(rq);  in __igt_reset_evict_vma()
1076 	if (!wait_until_running(&h, rq)) {  in __igt_reset_evict_vma()
1080 		       __func__, rq->fence.seqno, hws_seqno(&h, rq));  in __igt_reset_evict_vma()
1081 		intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);  in __igt_reset_evict_vma()
1098 	if (wait_for(waitqueue_active(&rq->execute), 10)) {  in __igt_reset_evict_vma()
1102 		intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);  in __igt_reset_evict_vma()
1109 	fake_hangcheck(rq, intel_engine_flag(rq->engine));  in __igt_reset_evict_vma()
1121 	i915_request_put(rq);  in __igt_reset_evict_vma()
1216 			struct i915_request *rq;  in igt_reset_queue()  local
1219 			rq = hang_create_request(&h, engine);  in igt_reset_queue()
1220 			if (IS_ERR(rq)) {  in igt_reset_queue()
1221 				err = PTR_ERR(rq);  in igt_reset_queue()
1225 			i915_request_get(rq);  in igt_reset_queue()
1226 			i915_request_add(rq);  in igt_reset_queue()
1242 				i915_request_put(rq);  in igt_reset_queue()
1259 				i915_request_put(rq);  in igt_reset_queue()
1278 				i915_request_put(rq);  in igt_reset_queue()
1284 			if (rq->fence.error) {  in igt_reset_queue()
1286 				       rq->fence.error);  in igt_reset_queue()
1287 				i915_request_put(rq);  in igt_reset_queue()
1295 				i915_request_put(rq);  in igt_reset_queue()
1302 			prev = rq;  in igt_reset_queue()
1334 	struct i915_request *rq;  in igt_handle_error()  local
1352 	rq = hang_create_request(&h, engine);  in igt_handle_error()
1353 	if (IS_ERR(rq)) {  in igt_handle_error()
1354 		err = PTR_ERR(rq);  in igt_handle_error()
1358 	i915_request_get(rq);  in igt_handle_error()
1359 	i915_request_add(rq);  in igt_handle_error()
1361 	if (!wait_until_running(&h, rq)) {  in igt_handle_error()
1365 		       __func__, rq->fence.seqno, hws_seqno(&h, rq));  in igt_handle_error()
1366 		intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);  in igt_handle_error()
1385 	if (rq->fence.error != -EIO) {  in igt_handle_error()
1392 	i915_request_put(rq);  in igt_handle_error()