24 static struct page *hwsp_page(struct intel_timeline *tl)  in hwsp_page()  argument
26 	struct drm_i915_gem_object *obj = tl->hwsp_ggtt->obj;  in hwsp_page()
32 static unsigned long hwsp_cacheline(struct intel_timeline *tl)  in hwsp_cacheline()  argument
34 	unsigned long address = (unsigned long)page_address(hwsp_page(tl));  in hwsp_cacheline()
36 	return (address + tl->hwsp_offset) / CACHELINE_BYTES;  in hwsp_cacheline()
55 			       struct intel_timeline *tl)  in __mock_hwsp_record()  argument
57 	tl = xchg(&state->history[idx], tl);  in __mock_hwsp_record()
58 	if (tl) {  in __mock_hwsp_record()
59 		radix_tree_delete(&state->cachelines, hwsp_cacheline(tl));  in __mock_hwsp_record()
60 		intel_timeline_put(tl);  in __mock_hwsp_record()
68 	struct intel_timeline *tl;  in __mock_hwsp_timeline()  local
75 		tl = intel_timeline_create(state->gt);  in __mock_hwsp_timeline()
76 		if (IS_ERR(tl))  in __mock_hwsp_timeline()
77 			return PTR_ERR(tl);  in __mock_hwsp_timeline()
79 		cacheline = hwsp_cacheline(tl);  in __mock_hwsp_timeline()
80 		err = radix_tree_insert(&state->cachelines, cacheline, tl);  in __mock_hwsp_timeline()
86 			intel_timeline_put(tl);  in __mock_hwsp_timeline()
91 		__mock_hwsp_record(state, idx, tl);  in __mock_hwsp_timeline()
172 static int __igt_sync(struct intel_timeline *tl,  in __igt_sync()  argument
179 	if (__intel_timeline_sync_is_later(tl, ctx, p->seqno) != p->expected) {  in __igt_sync()
186 		ret = __intel_timeline_sync_set(tl, ctx, p->seqno);  in __igt_sync()
214 	struct intel_timeline tl;  in igt_sync()  local
218 	mock_timeline_init(&tl, 0);  in igt_sync()
224 				ret = __igt_sync(&tl, ctx, p, "1");  in igt_sync()
230 	mock_timeline_fini(&tl);  in igt_sync()
232 	mock_timeline_init(&tl, 0);  in igt_sync()
238 				ret = __igt_sync(&tl, ctx, p, "2");  in igt_sync()
246 	mock_timeline_fini(&tl);  in igt_sync()
258 	struct intel_timeline tl;  in bench_sync()  local
264 	mock_timeline_init(&tl, 0);  in bench_sync()
296 		__intel_timeline_sync_set(&tl, id, 0);  in bench_sync()
311 		if (!__intel_timeline_sync_is_later(&tl, id, 0)) {  in bench_sync()
312 			mock_timeline_fini(&tl);  in bench_sync()
322 	mock_timeline_fini(&tl);  in bench_sync()
325 	mock_timeline_init(&tl, 0);  in bench_sync()
332 		__intel_timeline_sync_set(&tl, count++, 0);  in bench_sync()
342 		if (!__intel_timeline_sync_is_later(&tl, end_time, 0)) {  in bench_sync()
344 			mock_timeline_fini(&tl);  in bench_sync()
352 	mock_timeline_fini(&tl);  in bench_sync()
355 	mock_timeline_init(&tl, 0);  in bench_sync()
366 		if (!__intel_timeline_sync_is_later(&tl, id, seqno))  in bench_sync()
367 			__intel_timeline_sync_set(&tl, id, seqno);  in bench_sync()
375 	mock_timeline_fini(&tl);  in bench_sync()
383 		mock_timeline_init(&tl, 0);  in bench_sync()
395 			__intel_timeline_sync_is_later(&tl, id, 0);  in bench_sync()
396 			__intel_timeline_sync_set(&tl, id, 0);  in bench_sync()
404 		mock_timeline_fini(&tl);  in bench_sync()
453 tl_write(struct intel_timeline *tl, struct intel_engine_cs *engine, u32 value)  in tl_write()  argument
458 	err = intel_timeline_pin(tl, NULL);  in tl_write()
470 	err = emit_ggtt_store_dw(rq, tl->hwsp_offset, value);  in tl_write()
478 	intel_timeline_unpin(tl);  in tl_write()
488 	struct intel_timeline *tl;  in checked_intel_timeline_create()  local
490 	tl = intel_timeline_create(gt);  in checked_intel_timeline_create()
491 	if (IS_ERR(tl))  in checked_intel_timeline_create()
492 		return tl;  in checked_intel_timeline_create()
494 	if (READ_ONCE(*tl->hwsp_seqno) != tl->seqno) {  in checked_intel_timeline_create()
496 		       *tl->hwsp_seqno, tl->seqno);  in checked_intel_timeline_create()
497 		intel_timeline_put(tl);  in checked_intel_timeline_create()
501 	return tl;  in checked_intel_timeline_create()
533 			struct intel_timeline *tl;  in live_hwsp_engine()  local
536 			tl = checked_intel_timeline_create(gt);  in live_hwsp_engine()
537 			if (IS_ERR(tl)) {  in live_hwsp_engine()
538 				err = PTR_ERR(tl);  in live_hwsp_engine()
542 			rq = tl_write(tl, engine, count);  in live_hwsp_engine()
544 				intel_timeline_put(tl);  in live_hwsp_engine()
549 			timelines[count++] = tl;  in live_hwsp_engine()
562 		struct intel_timeline *tl = timelines[n];  in live_hwsp_engine()  local
564 		if (!err && READ_ONCE(*tl->hwsp_seqno) != n) {  in live_hwsp_engine()
566 				      n, tl->fence_context, tl->hwsp_offset, *tl->hwsp_seqno);  in live_hwsp_engine()
570 		intel_timeline_put(tl);  in live_hwsp_engine()
603 			struct intel_timeline *tl;  in live_hwsp_alternate()  local
609 			tl = checked_intel_timeline_create(gt);  in live_hwsp_alternate()
610 			if (IS_ERR(tl)) {  in live_hwsp_alternate()
611 				err = PTR_ERR(tl);  in live_hwsp_alternate()
616 			rq = tl_write(tl, engine, count);  in live_hwsp_alternate()
619 				intel_timeline_put(tl);  in live_hwsp_alternate()
624 			timelines[count++] = tl;  in live_hwsp_alternate()
634 		struct intel_timeline *tl = timelines[n];  in live_hwsp_alternate()  local
636 		if (!err && READ_ONCE(*tl->hwsp_seqno) != n) {  in live_hwsp_alternate()
638 				      n, tl->fence_context, tl->hwsp_offset, *tl->hwsp_seqno);  in live_hwsp_alternate()
642 		intel_timeline_put(tl);  in live_hwsp_alternate()
654 	struct intel_timeline *tl;  in live_hwsp_wrap()  local
663 	tl = intel_timeline_create(gt);  in live_hwsp_wrap()
664 	if (IS_ERR(tl))  in live_hwsp_wrap()
665 		return PTR_ERR(tl);  in live_hwsp_wrap()
667 	if (!tl->has_initial_breadcrumb || !tl->hwsp_cacheline)  in live_hwsp_wrap()
670 	err = intel_timeline_pin(tl, NULL);  in live_hwsp_wrap()
688 		tl->seqno = -4u;  in live_hwsp_wrap()
690 		mutex_lock_nested(&tl->mutex, SINGLE_DEPTH_NESTING);  in live_hwsp_wrap()
691 		err = intel_timeline_get_seqno(tl, rq, &seqno[0]);  in live_hwsp_wrap()
692 		mutex_unlock(&tl->mutex);  in live_hwsp_wrap()
698 			 seqno[0], tl->hwsp_offset);  in live_hwsp_wrap()
700 		err = emit_ggtt_store_dw(rq, tl->hwsp_offset, seqno[0]);  in live_hwsp_wrap()
705 		hwsp_seqno[0] = tl->hwsp_seqno;  in live_hwsp_wrap()
707 		mutex_lock_nested(&tl->mutex, SINGLE_DEPTH_NESTING);  in live_hwsp_wrap()
708 		err = intel_timeline_get_seqno(tl, rq, &seqno[1]);  in live_hwsp_wrap()
709 		mutex_unlock(&tl->mutex);  in live_hwsp_wrap()
715 			 seqno[1], tl->hwsp_offset);  in live_hwsp_wrap()
717 		err = emit_ggtt_store_dw(rq, tl->hwsp_offset, seqno[1]);  in live_hwsp_wrap()
722 		hwsp_seqno[1] = tl->hwsp_seqno;  in live_hwsp_wrap()
752 	intel_timeline_unpin(tl);  in live_hwsp_wrap()
754 	intel_timeline_put(tl);  in live_hwsp_wrap()
772 		struct intel_timeline *tl = ce->timeline;  in live_hwsp_rollover_kernel()  local
782 		GEM_BUG_ON(i915_active_fence_isset(&tl->last_request));  in live_hwsp_rollover_kernel()
783 		tl->seqno = 0;  in live_hwsp_rollover_kernel()
784 		timeline_rollback(tl);  in live_hwsp_rollover_kernel()
785 		timeline_rollback(tl);  in live_hwsp_rollover_kernel()
786 		WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno);  in live_hwsp_rollover_kernel()
801 			GEM_BUG_ON(rcu_access_pointer(this->timeline) != tl);  in live_hwsp_rollover_kernel()
852 		struct intel_timeline *tl;  in live_hwsp_rollover_user()  local
864 		tl = ce->timeline;  in live_hwsp_rollover_user()
865 		if (!tl->has_initial_breadcrumb || !tl->hwsp_cacheline)  in live_hwsp_rollover_user()
868 		timeline_rollback(tl);  in live_hwsp_rollover_user()
869 		timeline_rollback(tl);  in live_hwsp_rollover_user()
870 		WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno);  in live_hwsp_rollover_user()
885 			GEM_BUG_ON(rcu_access_pointer(this->timeline) != tl);  in live_hwsp_rollover_user()
946 			struct intel_timeline *tl;  in live_hwsp_recycle()  local
949 			tl = checked_intel_timeline_create(gt);  in live_hwsp_recycle()
950 			if (IS_ERR(tl)) {  in live_hwsp_recycle()
951 				err = PTR_ERR(tl);  in live_hwsp_recycle()
955 			rq = tl_write(tl, engine, count);  in live_hwsp_recycle()
957 				intel_timeline_put(tl);  in live_hwsp_recycle()
965 				intel_timeline_put(tl);  in live_hwsp_recycle()
970 			if (READ_ONCE(*tl->hwsp_seqno) != count) {  in live_hwsp_recycle()
972 					      count, tl->fence_context,  in live_hwsp_recycle()
973 					      tl->hwsp_offset, *tl->hwsp_seqno);  in live_hwsp_recycle()
979 			intel_timeline_put(tl);  in live_hwsp_recycle()