1830 static int kvm_hv_get_hc_data(struct kvm *kvm, struct kvm_hv_hcall *hc,  in kvm_hv_get_hc_data()  argument
1841 	if (hc->fast) {  in kvm_hv_get_hc_data()
1846 		if (orig_cnt > 2 * HV_HYPERCALL_MAX_XMM_REGISTERS - hc->consumed_xmm_halves)  in kvm_hv_get_hc_data()
1850 			j = i + hc->consumed_xmm_halves;  in kvm_hv_get_hc_data()
1852 				data[i] = sse128_hi(hc->xmm[j / 2]);  in kvm_hv_get_hc_data()
1854 				data[i] = sse128_lo(hc->xmm[j / 2]);  in kvm_hv_get_hc_data()
1859 	return kvm_read_guest(kvm, hc->ingpa + hc->data_offset, data,  in kvm_hv_get_hc_data()
1863 static u64 kvm_get_sparse_vp_set(struct kvm *kvm, struct kvm_hv_hcall *hc,  in kvm_get_sparse_vp_set()  argument
1866 	if (hc->var_cnt > HV_MAX_SPARSE_VCPU_BANKS)  in kvm_get_sparse_vp_set()
1870 	return kvm_hv_get_hc_data(kvm, hc, hc->var_cnt, KVM_HV_MAX_SPARSE_VCPU_SET_BITS,  in kvm_get_sparse_vp_set()
1874 static int kvm_hv_get_tlb_flush_entries(struct kvm *kvm, struct kvm_hv_hcall *hc, u64 entries[])  in kvm_hv_get_tlb_flush_entries()  argument
1876 	return kvm_hv_get_hc_data(kvm, hc, hc->rep_cnt, hc->rep_cnt, entries);  in kvm_hv_get_tlb_flush_entries()
1949 static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)  in kvm_hv_flush_tlb()  argument
1985 	if (!hc->fast && is_guest_mode(vcpu)) {  in kvm_hv_flush_tlb()
1986 		hc->ingpa = translate_nested_gpa(vcpu, hc->ingpa, 0, NULL);  in kvm_hv_flush_tlb()
1987 		if (unlikely(hc->ingpa == INVALID_GPA))  in kvm_hv_flush_tlb()
1991 	if (hc->code == HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST ||  in kvm_hv_flush_tlb()
1992 	    hc->code == HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE) {  in kvm_hv_flush_tlb()
1993 		if (hc->fast) {  in kvm_hv_flush_tlb()
1994 			flush.address_space = hc->ingpa;  in kvm_hv_flush_tlb()
1995 			flush.flags = hc->outgpa;  in kvm_hv_flush_tlb()
1996 			flush.processor_mask = sse128_lo(hc->xmm[0]);  in kvm_hv_flush_tlb()
1997 			hc->consumed_xmm_halves = 1;  in kvm_hv_flush_tlb()
1999 			if (unlikely(kvm_read_guest(kvm, hc->ingpa,  in kvm_hv_flush_tlb()
2002 			hc->data_offset = sizeof(flush);  in kvm_hv_flush_tlb()
2022 		if (hc->fast) {  in kvm_hv_flush_tlb()
2023 			flush_ex.address_space = hc->ingpa;  in kvm_hv_flush_tlb()
2024 			flush_ex.flags = hc->outgpa;  in kvm_hv_flush_tlb()
2026 			       &hc->xmm[0], sizeof(hc->xmm[0]));  in kvm_hv_flush_tlb()
2027 			hc->consumed_xmm_halves = 2;  in kvm_hv_flush_tlb()
2029 			if (unlikely(kvm_read_guest(kvm, hc->ingpa, &flush_ex,  in kvm_hv_flush_tlb()
2032 			hc->data_offset = sizeof(flush_ex);  in kvm_hv_flush_tlb()
2044 		if (hc->var_cnt != hweight64(valid_bank_mask))  in kvm_hv_flush_tlb()
2048 			if (!hc->var_cnt)  in kvm_hv_flush_tlb()
2051 			if (kvm_get_sparse_vp_set(kvm, hc, sparse_banks))  in kvm_hv_flush_tlb()
2062 		if (hc->fast)  in kvm_hv_flush_tlb()
2063 			hc->consumed_xmm_halves += hc->var_cnt;  in kvm_hv_flush_tlb()
2065 			hc->data_offset += hc->var_cnt * sizeof(sparse_banks[0]);  in kvm_hv_flush_tlb()
2068 	if (hc->code == HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE ||  in kvm_hv_flush_tlb()
2069 	    hc->code == HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX ||  in kvm_hv_flush_tlb()
2070 	    hc->rep_cnt > ARRAY_SIZE(__tlb_flush_entries)) {  in kvm_hv_flush_tlb()
2073 		if (kvm_hv_get_tlb_flush_entries(kvm, hc, __tlb_flush_entries))  in kvm_hv_flush_tlb()
2086 					     tlb_flush_entries, hc->rep_cnt);  in kvm_hv_flush_tlb()
2099 					     tlb_flush_entries, hc->rep_cnt);  in kvm_hv_flush_tlb()
2132 					     tlb_flush_entries, hc->rep_cnt);  in kvm_hv_flush_tlb()
2141 		((u64)hc->rep_cnt << HV_HYPERCALL_REP_COMP_OFFSET);  in kvm_hv_flush_tlb()
2165 static u64 kvm_hv_send_ipi(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)  in kvm_hv_send_ipi()  argument
2176 	if (hc->code == HVCALL_SEND_IPI) {  in kvm_hv_send_ipi()
2177 		if (!hc->fast) {  in kvm_hv_send_ipi()
2178 			if (unlikely(kvm_read_guest(kvm, hc->ingpa, &send_ipi,  in kvm_hv_send_ipi()
2185 			if (unlikely(hc->ingpa >> 32 != 0))  in kvm_hv_send_ipi()
2187 			sparse_banks[0] = hc->outgpa;  in kvm_hv_send_ipi()
2188 			vector = (u32)hc->ingpa;  in kvm_hv_send_ipi()
2195 		if (!hc->fast) {  in kvm_hv_send_ipi()
2196 			if (unlikely(kvm_read_guest(kvm, hc->ingpa, &send_ipi_ex,  in kvm_hv_send_ipi()
2200 			send_ipi_ex.vector = (u32)hc->ingpa;  in kvm_hv_send_ipi()
2201 			send_ipi_ex.vp_set.format = hc->outgpa;  in kvm_hv_send_ipi()
2202 			send_ipi_ex.vp_set.valid_bank_mask = sse128_lo(hc->xmm[0]);  in kvm_hv_send_ipi()
2213 		if (hc->var_cnt != hweight64(valid_bank_mask))  in kvm_hv_send_ipi()
2219 		if (!hc->var_cnt)  in kvm_hv_send_ipi()
2222 		if (!hc->fast)  in kvm_hv_send_ipi()
2223 			hc->data_offset = offsetof(struct hv_send_ipi_ex,  in kvm_hv_send_ipi()
2226 			hc->consumed_xmm_halves = 1;  in kvm_hv_send_ipi()
2228 		if (kvm_get_sparse_vp_set(kvm, hc, sparse_banks))  in kvm_hv_send_ipi()
2352 static u16 kvm_hvcall_signal_event(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)  in kvm_hvcall_signal_event()  argument
2357 	if (unlikely(!hc->fast)) {  in kvm_hvcall_signal_event()
2359 		gpa_t gpa = hc->ingpa;  in kvm_hvcall_signal_event()
2361 		if ((gpa & (__alignof__(hc->ingpa) - 1)) ||  in kvm_hvcall_signal_event()
2362 		    offset_in_page(gpa) + sizeof(hc->ingpa) > PAGE_SIZE)  in kvm_hvcall_signal_event()
2366 					  &hc->ingpa, sizeof(hc->ingpa));  in kvm_hvcall_signal_event()
2376 	if (hc->ingpa & 0xffff00000000ULL)  in kvm_hvcall_signal_event()
2379 	if (hc->ingpa & ~KVM_HYPERV_CONN_ID_MASK)  in kvm_hvcall_signal_event()
2384 	eventfd = idr_find(&hv->conn_to_evt, hc->ingpa);  in kvm_hvcall_signal_event()
2393 static bool is_xmm_fast_hypercall(struct kvm_hv_hcall *hc)  in is_xmm_fast_hypercall()  argument
2395 	switch (hc->code) {  in is_xmm_fast_hypercall()
2407 static void kvm_hv_hypercall_read_xmm(struct kvm_hv_hcall *hc)  in kvm_hv_hypercall_read_xmm()  argument
2413 		_kvm_read_sse_reg(reg, &hc->xmm[reg]);  in kvm_hv_hypercall_read_xmm()
2470 	struct kvm_hv_hcall hc;  in kvm_hv_hypercall()  local
2484 		hc.param = kvm_rcx_read(vcpu);  in kvm_hv_hypercall()
2485 		hc.ingpa = kvm_rdx_read(vcpu);  in kvm_hv_hypercall()
2486 		hc.outgpa = kvm_r8_read(vcpu);  in kvm_hv_hypercall()
2490 		hc.param = ((u64)kvm_rdx_read(vcpu) << 32) |  in kvm_hv_hypercall()
2492 		hc.ingpa = ((u64)kvm_rbx_read(vcpu) << 32) |  in kvm_hv_hypercall()
2494 		hc.outgpa = ((u64)kvm_rdi_read(vcpu) << 32) |  in kvm_hv_hypercall()
2498 	hc.code = hc.param & 0xffff;  in kvm_hv_hypercall()
2499 	hc.var_cnt = (hc.param & HV_HYPERCALL_VARHEAD_MASK) >> HV_HYPERCALL_VARHEAD_OFFSET;  in kvm_hv_hypercall()
2500 	hc.fast = !!(hc.param & HV_HYPERCALL_FAST_BIT);  in kvm_hv_hypercall()
2501 	hc.rep_cnt = (hc.param >> HV_HYPERCALL_REP_COMP_OFFSET) & 0xfff;  in kvm_hv_hypercall()
2502 	hc.rep_idx = (hc.param >> HV_HYPERCALL_REP_START_OFFSET) & 0xfff;  in kvm_hv_hypercall()
2503 	hc.rep = !!(hc.rep_cnt || hc.rep_idx);  in kvm_hv_hypercall()
2505 	trace_kvm_hv_hypercall(hc.code, hc.fast, hc.var_cnt, hc.rep_cnt,  in kvm_hv_hypercall()
2506 			       hc.rep_idx, hc.ingpa, hc.outgpa);  in kvm_hv_hypercall()
2508 	if (unlikely(!hv_check_hypercall_access(hv_vcpu, hc.code))) {  in kvm_hv_hypercall()
2513 	if (unlikely(hc.param & HV_HYPERCALL_RSVD_MASK)) {  in kvm_hv_hypercall()
2518 	if (hc.fast && is_xmm_fast_hypercall(&hc)) {  in kvm_hv_hypercall()
2526 		kvm_hv_hypercall_read_xmm(&hc);  in kvm_hv_hypercall()
2529 	switch (hc.code) {  in kvm_hv_hypercall()
2531 		if (unlikely(hc.rep || hc.var_cnt)) {  in kvm_hv_hypercall()
2538 		if (unlikely(hc.rep || hc.var_cnt)) {  in kvm_hv_hypercall()
2542 		ret = kvm_hvcall_signal_event(vcpu, &hc);  in kvm_hv_hypercall()
2548 		if (unlikely(hc.rep || hc.var_cnt || !to_hv_synic(vcpu)->active)) {  in kvm_hv_hypercall()
2554 		if (unlikely(hc.var_cnt)) {  in kvm_hv_hypercall()
2560 		if (unlikely(!hc.rep_cnt || hc.rep_idx)) {  in kvm_hv_hypercall()
2564 		ret = kvm_hv_flush_tlb(vcpu, &hc);  in kvm_hv_hypercall()
2567 		if (unlikely(hc.var_cnt)) {  in kvm_hv_hypercall()
2573 		if (unlikely(hc.rep)) {  in kvm_hv_hypercall()
2577 		ret = kvm_hv_flush_tlb(vcpu, &hc);  in kvm_hv_hypercall()
2580 		if (unlikely(hc.var_cnt)) {  in kvm_hv_hypercall()
2586 		if (unlikely(hc.rep)) {  in kvm_hv_hypercall()
2590 		ret = kvm_hv_send_ipi(vcpu, &hc);  in kvm_hv_hypercall()
2594 		if (unlikely(hc.fast)) {  in kvm_hv_hypercall()
2614 		if (unlikely(hc.fast)) {  in kvm_hv_hypercall()
2630 	vcpu->run->hyperv.u.hcall.input = hc.param;  in kvm_hv_hypercall()
2631 	vcpu->run->hyperv.u.hcall.params[0] = hc.ingpa;  in kvm_hv_hypercall()
2632 	vcpu->run->hyperv.u.hcall.params[1] = hc.outgpa;  in kvm_hv_hypercall()