1368 static void i40iw_copy_user_pgaddrs(struct i40iw_mr *iwmr,  in i40iw_copy_user_pgaddrs()  argument
1372 	struct ib_umem *region = iwmr->region;  in i40iw_copy_user_pgaddrs()
1373 	struct i40iw_pbl *iwpbl = &iwmr->iwpbl;  in i40iw_copy_user_pgaddrs()
1385 		if ((iwmr->type == IW_MEMREG_TYPE_QP) &&  in i40iw_copy_user_pgaddrs()
1393 				*pbl = cpu_to_le64(pg_addr & iwmr->page_msk);  in i40iw_copy_user_pgaddrs()
1394 			else if (!(pg_addr & ~iwmr->page_msk))  in i40iw_copy_user_pgaddrs()
1408 static void i40iw_set_hugetlb_values(u64 addr, struct i40iw_mr *iwmr)  in i40iw_set_hugetlb_values()  argument
1418 			iwmr->page_size = huge_page_size(h);  in i40iw_set_hugetlb_values()
1419 			iwmr->page_msk = huge_page_mask(h);  in i40iw_set_hugetlb_values()
1484 			     struct i40iw_mr *iwmr,  in i40iw_setup_pbles()  argument
1487 	struct i40iw_pbl *iwpbl = &iwmr->iwpbl;  in i40iw_setup_pbles()
1496 		status = i40iw_get_pble(&iwdev->sc_dev, iwdev->pble_rsrc, palloc, iwmr->page_cnt);  in i40iw_setup_pbles()
1506 		pbl = iwmr->pgaddrmem;  in i40iw_setup_pbles()
1509 	i40iw_copy_user_pgaddrs(iwmr, pbl, level);  in i40iw_setup_pbles()
1512 		iwmr->pgaddrmem[0] = *pbl;  in i40iw_setup_pbles()
1530 	struct i40iw_mr *iwmr = iwpbl->iwmr;  in i40iw_handle_q_mem()  local
1534 	u64 *arr = iwmr->pgaddrmem;  in i40iw_handle_q_mem()
1541 	pg_size = iwmr->page_size;  in i40iw_handle_q_mem()
1543 	err = i40iw_setup_pbles(iwdev, iwmr, use_pbles);  in i40iw_handle_q_mem()
1556 	if (iwmr->type == IW_MEMREG_TYPE_QP) {  in i40iw_handle_q_mem()
1601 static int i40iw_hw_alloc_stag(struct i40iw_device *iwdev, struct i40iw_mr *iwmr)  in i40iw_hw_alloc_stag()  argument
1604 	struct i40iw_pd *iwpd = to_iwpd(iwmr->ibmr.pd);  in i40iw_hw_alloc_stag()
1618 	info->stag_idx = iwmr->stag >> I40IW_CQPSQ_STAG_IDX_SHIFT;  in i40iw_hw_alloc_stag()
1620 	info->total_len = iwmr->length;  in i40iw_hw_alloc_stag()
1649 	struct i40iw_mr *iwmr;  in i40iw_alloc_mr()  local
1654 	iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);  in i40iw_alloc_mr()
1655 	if (!iwmr)  in i40iw_alloc_mr()
1664 	iwmr->stag = stag;  in i40iw_alloc_mr()
1665 	iwmr->ibmr.rkey = stag;  in i40iw_alloc_mr()
1666 	iwmr->ibmr.lkey = stag;  in i40iw_alloc_mr()
1667 	iwmr->ibmr.pd = pd;  in i40iw_alloc_mr()
1668 	iwmr->ibmr.device = pd->device;  in i40iw_alloc_mr()
1669 	iwpbl = &iwmr->iwpbl;  in i40iw_alloc_mr()
1670 	iwpbl->iwmr = iwmr;  in i40iw_alloc_mr()
1671 	iwmr->type = IW_MEMREG_TYPE_MEM;  in i40iw_alloc_mr()
1673 	iwmr->page_cnt = max_num_sg;  in i40iw_alloc_mr()
1675 	status = i40iw_get_pble(&iwdev->sc_dev, iwdev->pble_rsrc, palloc, iwmr->page_cnt);  in i40iw_alloc_mr()
1682 	err_code = i40iw_hw_alloc_stag(iwdev, iwmr);  in i40iw_alloc_mr()
1687 	return &iwmr->ibmr;  in i40iw_alloc_mr()
1693 	kfree(iwmr);  in i40iw_alloc_mr()
1704 	struct i40iw_mr *iwmr = to_iwmr(ibmr);  in i40iw_set_page()  local
1705 	struct i40iw_pbl *iwpbl = &iwmr->iwpbl;  in i40iw_set_page()
1709 	if (unlikely(iwmr->npages == iwmr->page_cnt))  in i40iw_set_page()
1713 	pbl[iwmr->npages++] = cpu_to_le64(addr);  in i40iw_set_page()
1726 	struct i40iw_mr *iwmr = to_iwmr(ibmr);  in i40iw_map_mr_sg()  local
1728 	iwmr->npages = 0;  in i40iw_map_mr_sg()
1765 			  struct i40iw_mr *iwmr,  in i40iw_hwreg_mr()  argument
1768 	struct i40iw_pbl *iwpbl = &iwmr->iwpbl;  in i40iw_hwreg_mr()
1770 	struct i40iw_pd *iwpd = to_iwpd(iwmr->ibmr.pd);  in i40iw_hwreg_mr()
1785 	stag_info->stag_idx = iwmr->stag >> I40IW_CQPSQ_STAG_IDX_SHIFT;  in i40iw_hwreg_mr()
1786 	stag_info->stag_key = (u8)iwmr->stag;  in i40iw_hwreg_mr()
1787 	stag_info->total_len = iwmr->length;  in i40iw_hwreg_mr()
1791 	stag_info->page_size = iwmr->page_size;  in i40iw_hwreg_mr()
1802 		stag_info->reg_addr_pa = iwmr->pgaddrmem[0];  in i40iw_hwreg_mr()
1839 	struct i40iw_mr *iwmr;  in i40iw_reg_user_mr()  local
1866 	iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);  in i40iw_reg_user_mr()
1867 	if (!iwmr) {  in i40iw_reg_user_mr()
1872 	iwpbl = &iwmr->iwpbl;  in i40iw_reg_user_mr()
1873 	iwpbl->iwmr = iwmr;  in i40iw_reg_user_mr()
1874 	iwmr->region = region;  in i40iw_reg_user_mr()
1875 	iwmr->ibmr.pd = pd;  in i40iw_reg_user_mr()
1876 	iwmr->ibmr.device = pd->device;  in i40iw_reg_user_mr()
1879 	iwmr->page_size = PAGE_SIZE;  in i40iw_reg_user_mr()
1880 	iwmr->page_msk = PAGE_MASK;  in i40iw_reg_user_mr()
1883 		i40iw_set_hugetlb_values(start, iwmr);  in i40iw_reg_user_mr()
1885 	region_length = region->length + (start & (iwmr->page_size - 1));  in i40iw_reg_user_mr()
1886 	pg_shift = ffs(iwmr->page_size) - 1;  in i40iw_reg_user_mr()
1888 	pbl_depth += (region_length & (iwmr->page_size - 1)) ? 1 : 0;  in i40iw_reg_user_mr()
1889 	iwmr->length = region->length;  in i40iw_reg_user_mr()
1894 	iwmr->type = req.reg_type;  in i40iw_reg_user_mr()
1895 	iwmr->page_cnt = (u32)pbl_depth;  in i40iw_reg_user_mr()
1920 		use_pbles = (iwmr->page_cnt != 1);  in i40iw_reg_user_mr()
1923 		err = i40iw_setup_pbles(iwdev, iwmr, use_pbles);  in i40iw_reg_user_mr()
1928 			ret = i40iw_check_mr_contiguous(palloc, iwmr->page_size);  in i40iw_reg_user_mr()
1942 		iwmr->stag = stag;  in i40iw_reg_user_mr()
1943 		iwmr->ibmr.rkey = stag;  in i40iw_reg_user_mr()
1944 		iwmr->ibmr.lkey = stag;  in i40iw_reg_user_mr()
1946 		err = i40iw_hwreg_mr(iwdev, iwmr, access);  in i40iw_reg_user_mr()
1957 	iwmr->type = req.reg_type;  in i40iw_reg_user_mr()
1960 	return &iwmr->ibmr;  in i40iw_reg_user_mr()
1966 	kfree(iwmr);  in i40iw_reg_user_mr()
1987 	struct i40iw_mr *iwmr;  in i40iw_reg_phys_mr()  local
1993 	iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);  in i40iw_reg_phys_mr()
1994 	if (!iwmr)  in i40iw_reg_phys_mr()
1996 	iwmr->ibmr.pd = pd;  in i40iw_reg_phys_mr()
1997 	iwmr->ibmr.device = pd->device;  in i40iw_reg_phys_mr()
1998 	iwpbl = &iwmr->iwpbl;  in i40iw_reg_phys_mr()
1999 	iwpbl->iwmr = iwmr;  in i40iw_reg_phys_mr()
2000 	iwmr->type = IW_MEMREG_TYPE_MEM;  in i40iw_reg_phys_mr()
2008 	iwmr->stag = stag;  in i40iw_reg_phys_mr()
2009 	iwmr->ibmr.rkey = stag;  in i40iw_reg_phys_mr()
2010 	iwmr->ibmr.lkey = stag;  in i40iw_reg_phys_mr()
2011 	iwmr->page_cnt = 1;  in i40iw_reg_phys_mr()
2012 	iwmr->pgaddrmem[0]  = addr;  in i40iw_reg_phys_mr()
2013 	iwmr->length = size;  in i40iw_reg_phys_mr()
2014 	status = i40iw_hwreg_mr(iwdev, iwmr, access);  in i40iw_reg_phys_mr()
2022 	return &iwmr->ibmr;  in i40iw_reg_phys_mr()
2024 	kfree(iwmr);  in i40iw_reg_phys_mr()
2045 static void i40iw_del_memlist(struct i40iw_mr *iwmr,  in i40iw_del_memlist()  argument
2048 	struct i40iw_pbl *iwpbl = &iwmr->iwpbl;  in i40iw_del_memlist()
2051 	switch (iwmr->type) {  in i40iw_del_memlist()
2081 	struct i40iw_mr *iwmr = to_iwmr(ib_mr);  in i40iw_dereg_mr()  local
2085 	struct i40iw_pbl *iwpbl = &iwmr->iwpbl;  in i40iw_dereg_mr()
2091 	if (iwmr->region)  in i40iw_dereg_mr()
2092 		ib_umem_release(iwmr->region);  in i40iw_dereg_mr()
2094 	if (iwmr->type != IW_MEMREG_TYPE_MEM) {  in i40iw_dereg_mr()
2099 			i40iw_del_memlist(iwmr, ucontext);  in i40iw_dereg_mr()
2101 		if (iwpbl->pbl_allocated && iwmr->type != IW_MEMREG_TYPE_QP)  in i40iw_dereg_mr()
2103 		kfree(iwmr);  in i40iw_dereg_mr()
2130 	i40iw_free_stag(iwdev, iwmr->stag);  in i40iw_dereg_mr()
2133 	kfree(iwmr);  in i40iw_dereg_mr()
2322 			struct i40iw_mr *iwmr = to_iwmr(reg_wr(ib_wr)->mr);  in i40iw_post_send()  local
2324 			struct i40iw_pble_alloc *palloc = &iwmr->iwpbl.pble_alloc;  in i40iw_post_send()
2337 			info.va = (void *)(uintptr_t)iwmr->ibmr.iova;  in i40iw_post_send()
2338 			info.total_len = iwmr->ibmr.length;  in i40iw_post_send()
2344 			if (iwmr->npages > I40IW_MIN_PAGES_PER_FMR)  in i40iw_post_send()