26  * to get bus addresses of each of the SG entries and their lengths.
27  * You should only work with the number of sg entries dma_map_sg
28  * returns, or alternatively stop on the first sg_dma_len(sg) which
31 #define sg_dma_address(sg)	((sg)->dma_address)  argument
34 #define sg_dma_len(sg)		((sg)->dma_length)  argument
36 #define sg_dma_len(sg)		((sg)->length)  argument
52  * Notes on SG table design.
55  * the page pointer AND encode information about the sg table as well. The two
58  * If bit 0 is set, then the page_link contains a pointer to the next sg
59  * table list. Otherwise the next entry is at sg + 1.
61  * If bit 1 is set, then this sg entry is the last element in a list.
72  * a valid sg entry, or whether it points to the start of a new scatterlist.
77 static inline unsigned int __sg_flags(struct scatterlist *sg)  in __sg_flags()  argument
79 	return sg->page_link & SG_PAGE_LINK_MASK;  in __sg_flags()
82 static inline struct scatterlist *sg_chain_ptr(struct scatterlist *sg)  in sg_chain_ptr()  argument
84 	return (struct scatterlist *)(sg->page_link & ~SG_PAGE_LINK_MASK);  in sg_chain_ptr()
87 static inline bool sg_is_chain(struct scatterlist *sg)  in sg_is_chain()  argument
89 	return __sg_flags(sg) & SG_CHAIN;  in sg_is_chain()
92 static inline bool sg_is_last(struct scatterlist *sg)  in sg_is_last()  argument
94 	return __sg_flags(sg) & SG_END;  in sg_is_last()
98  * sg_assign_page - Assign a given page to an SG entry
99  * @sg:		    SG entry
103  *   Assign page to sg entry. Also see sg_set_page(), the most commonly used
107 static inline void sg_assign_page(struct scatterlist *sg, struct page *page)  in sg_assign_page()  argument
109 	unsigned long page_link = sg->page_link & (SG_CHAIN | SG_END);  in sg_assign_page()
117 	BUG_ON(sg_is_chain(sg));  in sg_assign_page()
119 	sg->page_link = page_link | (unsigned long) page;  in sg_assign_page()
123  * sg_set_page - Set sg entry to point at given page
124  * @sg:		 SG entry
130  *   Use this function to set an sg entry pointing at a page, never assign
131  *   the page directly. We encode sg table information in the lower bits
133  *   to an sg entry.
136 static inline void sg_set_page(struct scatterlist *sg, struct page *page,  in sg_set_page()  argument
139 	sg_assign_page(sg, page);  in sg_set_page()
140 	sg->offset = offset;  in sg_set_page()
141 	sg->length = len;  in sg_set_page()
144 static inline struct page *sg_page(struct scatterlist *sg)  in sg_page()  argument
147 	BUG_ON(sg_is_chain(sg));  in sg_page()
149 	return (struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK);  in sg_page()
153  * sg_set_buf - Set sg entry to point at given data
154  * @sg:		 SG entry
159 static inline void sg_set_buf(struct scatterlist *sg, const void *buf,  in sg_set_buf()  argument
165 	sg_set_page(sg, virt_to_page(buf), buflen, offset_in_page(buf));  in sg_set_buf()
169  * Loop over each sg element, following the pointer to a new list if necessary
171 #define for_each_sg(sglist, sg, nr, __i)	\  argument
172 	for (__i = 0, sg = (sglist); __i < (nr); __i++, sg = sg_next(sg))
175  * Loop over each sg element in the given sg_table object.
177 #define for_each_sgtable_sg(sgt, sg, i)		\  argument
178 	for_each_sg((sgt)->sgl, sg, (sgt)->orig_nents, i)
181  * Loop over each sg element in the given *DMA mapped* sg_table object.
182  * Please use sg_dma_address(sg) and sg_dma_len(sg) to extract DMA addresses
185 #define for_each_sgtable_dma_sg(sgt, sg, i)	\  argument
186 	for_each_sg((sgt)->sgl, sg, (sgt)->nents, i)
222  * @sg:		 SG entryScatterlist
225  *   Marks the passed in sg entry as the termination point for the sg
229 static inline void sg_mark_end(struct scatterlist *sg)  in sg_mark_end()  argument
234 	sg->page_link |= SG_END;  in sg_mark_end()
235 	sg->page_link &= ~SG_CHAIN;  in sg_mark_end()
240  * @sg:		 SG entryScatterlist
246 static inline void sg_unmark_end(struct scatterlist *sg)  in sg_unmark_end()  argument
248 	sg->page_link &= ~SG_END;  in sg_unmark_end()
264  * @sg:		 SG entry
270 static inline bool sg_is_dma_bus_address(struct scatterlist *sg)  in sg_is_dma_bus_address()  argument
272 	return sg->dma_flags & SG_DMA_BUS_ADDRESS;  in sg_is_dma_bus_address()
277  * @sg:		 SG entry
280  *   Marks the passed in sg entry to indicate that the dma_address is
285 static inline void sg_dma_mark_bus_address(struct scatterlist *sg)  in sg_dma_mark_bus_address()  argument
287 	sg->dma_flags |= SG_DMA_BUS_ADDRESS;  in sg_dma_mark_bus_address()
292  * @sg:		 SG entry
297 static inline void sg_dma_unmark_bus_address(struct scatterlist *sg)  in sg_dma_unmark_bus_address()  argument
299 	sg->dma_flags &= ~SG_DMA_BUS_ADDRESS;  in sg_dma_unmark_bus_address()
304 static inline bool sg_is_dma_bus_address(struct scatterlist *sg)  in sg_is_dma_bus_address()  argument
308 static inline void sg_dma_mark_bus_address(struct scatterlist *sg)  in sg_dma_mark_bus_address()  argument
311 static inline void sg_dma_unmark_bus_address(struct scatterlist *sg)  in sg_dma_unmark_bus_address()  argument
318  * sg_phys - Return physical address of an sg entry
319  * @sg:	     SG entry
322  *   This calls page_to_phys() on the page in this sg entry, and adds the
323  *   sg offset. The caller must know that it is legal to call page_to_phys()
324  *   on the sg page.
327 static inline dma_addr_t sg_phys(struct scatterlist *sg)  in sg_phys()  argument
329 	return page_to_phys(sg_page(sg)) + sg->offset;  in sg_phys()
333  * sg_virt - Return virtual address of an sg entry
334  * @sg:      SG entry
337  *   This calls page_address() on the page in this sg entry, and adds the
338  *   sg offset. The caller must know that the sg page has a valid virtual
342 static inline void *sg_virt(struct scatterlist *sg)  in sg_virt()  argument
344 	return page_address(sg_page(sg)) + sg->offset;  in sg_virt()
348  * sg_init_marker - Initialize markers in sg table
349  * @sgl:	   The SG table
359 int sg_nents(struct scatterlist *sg);
360 int sg_nents_for_len(struct scatterlist *sg, u64 len);
392  * sg_alloc_table_from_pages - Allocate and initialize an sg table from
394  * @sgt:	 The sg table header to use
402  *    Allocate and initialize an sg table from a list of pages. Contiguous
405  *    specified by the page array. The returned sg table is released by
454  * The maximum number of SG segments that we will put inside a
463  * Like SG_CHUNK_SIZE, but for archs that have sg chaining. This limit
481  * sg page iterator
483  * Iterates over sg entries page-by-page.  On each successful iteration, you
485  * @piter->sg will point to the sg holding this page and @piter->sg_pgoffset to
486  * the page's page offset within the sg. The iteration will stop either when a
487  * maximum number of sg entries was reached or a terminating sg
488  * (sg_last(sg) == true) was reached.
491 	struct scatterlist	*sg;		/* sg holding the page */  member
492 	unsigned int		sg_pgoffset;	/* page offset within the sg */
495 	unsigned int		__nents;	/* remaining sg entries */
501  * sg page iterator for DMA addresses
522 	return nth_page(sg_page(piter->sg), piter->sg_pgoffset);  in sg_page_iter_page()
533 	return sg_dma_address(dma_iter->base.sg) +  in sg_page_iter_dma_address()
538  * for_each_sg_page - iterate over the pages of the given sg list
540  * @piter:	page iterator to hold current page, sg, sg_pgoffset
541  * @nents:	maximum number of sg entries to iterate over
552  * for_each_sg_dma_page - iterate over the pages of the given sg list
555  * @dma_nents:	maximum number of sg entries to iterate over, this is the value
596  * Mapping sg iterator
598  * Iterates over sg entries mapping page-by-page.  On each successful