15 static inline u64 mic_system_page_mask(struct mic_device *mdev)  in mic_system_page_mask()  argument
17 	return (1ULL << mdev->smpt->info.page_shift) - 1ULL;  in mic_system_page_mask()
20 static inline u8 mic_sys_addr_to_smpt(struct mic_device *mdev, dma_addr_t pa)  in mic_sys_addr_to_smpt()  argument
22 	return (pa - mdev->smpt->info.base) >> mdev->smpt->info.page_shift;  in mic_sys_addr_to_smpt()
25 static inline u64 mic_smpt_to_pa(struct mic_device *mdev, u8 index)  in mic_smpt_to_pa()  argument
27 	return mdev->smpt->info.base + (index * mdev->smpt->info.page_size);  in mic_smpt_to_pa()
30 static inline u64 mic_smpt_offset(struct mic_device *mdev, dma_addr_t pa)  in mic_smpt_offset()  argument
32 	return pa & mic_system_page_mask(mdev);  in mic_smpt_offset()
35 static inline u64 mic_smpt_align_low(struct mic_device *mdev, dma_addr_t pa)  in mic_smpt_align_low()  argument
37 	return ALIGN(pa - mic_system_page_mask(mdev),  in mic_smpt_align_low()
38 		mdev->smpt->info.page_size);  in mic_smpt_align_low()
41 static inline u64 mic_smpt_align_high(struct mic_device *mdev, dma_addr_t pa)  in mic_smpt_align_high()  argument
43 	return ALIGN(pa, mdev->smpt->info.page_size);  in mic_smpt_align_high()
47 static inline u64 mic_max_system_memory(struct mic_device *mdev)  in mic_max_system_memory()  argument
49 	return mdev->smpt->info.num_reg * mdev->smpt->info.page_size;  in mic_max_system_memory()
53 static inline u64 mic_max_system_addr(struct mic_device *mdev)  in mic_max_system_addr()  argument
55 	return mdev->smpt->info.base + mic_max_system_memory(mdev) - 1ULL;  in mic_max_system_addr()
60 mic_is_system_addr(struct mic_device *mdev, dma_addr_t pa)  in mic_is_system_addr()  argument
62 	return pa >= mdev->smpt->info.base && pa <= mic_max_system_addr(mdev);  in mic_is_system_addr()
67 			       int entries, struct mic_device *mdev)  in mic_add_smpt_entry()  argument
69 	struct mic_smpt_info *smpt_info = mdev->smpt;  in mic_add_smpt_entry()
76 			mdev->smpt_ops->set(mdev, addr, i);  in mic_add_smpt_entry()
87 static dma_addr_t mic_smpt_op(struct mic_device *mdev, u64 dma_addr,  in mic_smpt_op()  argument
96 	struct mic_smpt_info *smpt_info = mdev->smpt;  in mic_smpt_op()
125 	mic_addr = mic_smpt_to_pa(mdev, spt);  in mic_smpt_op()
126 	mic_add_smpt_entry(spt, ref, dma_addr, entries, mdev);  in mic_smpt_op()
138 static int mic_get_smpt_ref_count(struct mic_device *mdev, dma_addr_t dma_addr,  in mic_get_smpt_ref_count()  argument
146 		ref[i++] = min(mic_smpt_align_high(mdev, start + 1),  in mic_get_smpt_ref_count()
148 		start = mic_smpt_align_high(mdev, start + 1);  in mic_get_smpt_ref_count()
152 		*smpt_start = mic_smpt_align_low(mdev, dma_addr);  in mic_get_smpt_ref_count()
165 dma_addr_t mic_to_dma_addr(struct mic_device *mdev, dma_addr_t mic_addr)  in mic_to_dma_addr()  argument
167 	struct mic_smpt_info *smpt_info = mdev->smpt;  in mic_to_dma_addr()
171 	if (!mic_is_system_addr(mdev, mic_addr)) {  in mic_to_dma_addr()
172 		dev_err(&mdev->pdev->dev,  in mic_to_dma_addr()
176 	spt = mic_sys_addr_to_smpt(mdev, mic_addr);  in mic_to_dma_addr()
178 		mic_smpt_offset(mdev, mic_addr);  in mic_to_dma_addr()
194 dma_addr_t mic_map(struct mic_device *mdev, dma_addr_t dma_addr, size_t size)  in mic_map()  argument
201 	if (!size || size > mic_max_system_memory(mdev))  in mic_map()
204 	ref = kmalloc_array(mdev->smpt->info.num_reg, sizeof(s64), GFP_ATOMIC);  in mic_map()
208 	num_entries = mic_get_smpt_ref_count(mdev, dma_addr, size,  in mic_map()
212 	mic_addr = mic_smpt_op(mdev, smpt_start, num_entries, ref, size);  in mic_map()
221 	if (!mic_addr && MIC_FAMILY_X100 == mdev->family) {  in mic_map()
222 		dev_err(&mdev->pdev->dev,  in mic_map()
227 		return mic_addr + mic_smpt_offset(mdev, dma_addr);  in mic_map()
242 void mic_unmap(struct mic_device *mdev, dma_addr_t mic_addr, size_t size)  in mic_unmap()  argument
244 	struct mic_smpt_info *smpt_info = mdev->smpt;  in mic_unmap()
254 	if (!mic_is_system_addr(mdev, mic_addr)) {  in mic_unmap()
255 		dev_err(&mdev->pdev->dev,  in mic_unmap()
260 	spt = mic_sys_addr_to_smpt(mdev, mic_addr);  in mic_unmap()
261 	ref = kmalloc_array(mdev->smpt->info.num_reg, sizeof(s64), GFP_ATOMIC);  in mic_unmap()
266 	num_smpt = mic_get_smpt_ref_count(mdev, mic_addr, size, ref, NULL);  in mic_unmap()
275 			dev_warn(&mdev->pdev->dev,  in mic_unmap()
295 dma_addr_t mic_map_single(struct mic_device *mdev, void *va, size_t size)  in mic_map_single()  argument
298 	struct pci_dev *pdev = mdev->pdev;  in mic_map_single()
303 		mic_addr = mic_map(mdev, dma_addr, size);  in mic_map_single()
305 			dev_err(&mdev->pdev->dev,  in mic_map_single()
327 mic_unmap_single(struct mic_device *mdev, dma_addr_t mic_addr, size_t size)  in mic_unmap_single()  argument
329 	struct pci_dev *pdev = mdev->pdev;  in mic_unmap_single()
330 	dma_addr_t dma_addr = mic_to_dma_addr(mdev, mic_addr);  in mic_unmap_single()
331 	mic_unmap(mdev, mic_addr, size);  in mic_unmap_single()
342 int mic_smpt_init(struct mic_device *mdev)  in mic_smpt_init()  argument
348 	mdev->smpt = kmalloc(sizeof(*mdev->smpt), GFP_KERNEL);  in mic_smpt_init()
349 	if (!mdev->smpt)  in mic_smpt_init()
352 	smpt_info = mdev->smpt;  in mic_smpt_init()
353 	mdev->smpt_ops->init(mdev);  in mic_smpt_init()
365 		mdev->smpt_ops->set(mdev, dma_addr, i);  in mic_smpt_init()
383 void mic_smpt_uninit(struct mic_device *mdev)  in mic_smpt_uninit()  argument
385 	struct mic_smpt_info *smpt_info = mdev->smpt;  in mic_smpt_uninit()
388 	dev_dbg(&mdev->pdev->dev,  in mic_smpt_uninit()
390 		mdev->id, smpt_info->ref_count,  in mic_smpt_uninit()
394 		dev_dbg(&mdev->pdev->dev,  in mic_smpt_uninit()
399 			dev_warn(&mdev->pdev->dev,  in mic_smpt_uninit()
418 void mic_smpt_restore(struct mic_device *mdev)  in mic_smpt_restore()  argument
423 	for (i = 0; i < mdev->smpt->info.num_reg; i++) {  in mic_smpt_restore()
424 		dma_addr = mdev->smpt->entry[i].dma_addr;  in mic_smpt_restore()
425 		mdev->smpt_ops->set(mdev, dma_addr, i);  in mic_smpt_restore()