blob: cb3c8fe6acd776f55f78ca9cebe05028b8c7cf70 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001#ifndef _LINUX_SCATTERLIST_H
2#define _LINUX_SCATTERLIST_H
3
Paul Gortmaker187f1882011-11-23 20:12:59 -05004#include <linux/string.h>
Christoph Hellwig84be4562015-05-01 12:46:15 +02005#include <linux/types.h>
Paul Gortmaker187f1882011-11-23 20:12:59 -05006#include <linux/bug.h>
7#include <linux/mm.h>
Jens Axboe18dabf42007-10-22 19:57:20 +02008#include <asm/io.h>
9
Christoph Hellwig84be4562015-05-01 12:46:15 +020010struct scatterlist {
11#ifdef CONFIG_DEBUG_SG
12 unsigned long sg_magic;
13#endif
14 unsigned long page_link;
15 unsigned int offset;
16 unsigned int length;
17 dma_addr_t dma_address;
18#ifdef CONFIG_NEED_SG_DMA_LENGTH
19 unsigned int dma_length;
20#endif
21};
22
23/*
24 * These macros should be used after a dma_map_sg call has been done
25 * to get bus addresses of each of the SG entries and their lengths.
26 * You should only work with the number of sg entries dma_map_sg
27 * returns, or alternatively stop on the first sg_dma_len(sg) which
28 * is 0.
29 */
30#define sg_dma_address(sg) ((sg)->dma_address)
31
32#ifdef CONFIG_NEED_SG_DMA_LENGTH
33#define sg_dma_len(sg) ((sg)->dma_length)
34#else
35#define sg_dma_len(sg) ((sg)->length)
36#endif
37
Jens Axboe0db92992007-11-30 09:16:50 +010038struct sg_table {
39 struct scatterlist *sgl; /* the list */
40 unsigned int nents; /* number of mapped entries */
41 unsigned int orig_nents; /* original size of list */
42};
43
Jens Axboe18dabf42007-10-22 19:57:20 +020044/*
45 * Notes on SG table design.
46 *
Christoph Hellwig84be4562015-05-01 12:46:15 +020047 * We use the unsigned long page_link field in the scatterlist struct to place
48 * the page pointer AND encode information about the sg table as well. The two
49 * lower bits are reserved for this information.
Jens Axboe18dabf42007-10-22 19:57:20 +020050 *
51 * If bit 0 is set, then the page_link contains a pointer to the next sg
52 * table list. Otherwise the next entry is at sg + 1.
53 *
54 * If bit 1 is set, then this sg entry is the last element in a list.
55 *
56 * See sg_next().
57 *
58 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070059
Jens Axboed6ec0842007-10-22 20:01:06 +020060#define SG_MAGIC 0x87654321
61
Tejun Heo645a8d92007-11-27 09:30:39 +010062/*
63 * We overload the LSB of the page pointer to indicate whether it's
64 * a valid sg entry, or whether it points to the start of a new scatterlist.
65 * Those low bits are there for everyone! (thanks mason :-)
66 */
67#define sg_is_chain(sg) ((sg)->page_link & 0x01)
68#define sg_is_last(sg) ((sg)->page_link & 0x02)
69#define sg_chain_ptr(sg) \
70 ((struct scatterlist *) ((sg)->page_link & ~0x03))
71
Jens Axboe82f66fb2007-10-22 17:07:37 +020072/**
Jens Axboe642f1492007-10-24 11:20:47 +020073 * sg_assign_page - Assign a given page to an SG entry
74 * @sg: SG entry
75 * @page: The page
Jens Axboe82f66fb2007-10-22 17:07:37 +020076 *
77 * Description:
Jens Axboe642f1492007-10-24 11:20:47 +020078 * Assign page to sg entry. Also see sg_set_page(), the most commonly used
79 * variant.
Jens Axboe82f66fb2007-10-22 17:07:37 +020080 *
81 **/
Jens Axboe642f1492007-10-24 11:20:47 +020082static inline void sg_assign_page(struct scatterlist *sg, struct page *page)
Jens Axboe82f66fb2007-10-22 17:07:37 +020083{
Jens Axboe18dabf42007-10-22 19:57:20 +020084 unsigned long page_link = sg->page_link & 0x3;
85
Jens Axboede261032007-10-23 20:35:58 +020086 /*
87 * In order for the low bit stealing approach to work, pages
88 * must be aligned at a 32-bit boundary as a minimum.
89 */
90 BUG_ON((unsigned long) page & 0x03);
Jens Axboed6ec0842007-10-22 20:01:06 +020091#ifdef CONFIG_DEBUG_SG
92 BUG_ON(sg->sg_magic != SG_MAGIC);
Tejun Heo645a8d92007-11-27 09:30:39 +010093 BUG_ON(sg_is_chain(sg));
Jens Axboed6ec0842007-10-22 20:01:06 +020094#endif
Jens Axboe18dabf42007-10-22 19:57:20 +020095 sg->page_link = page_link | (unsigned long) page;
Jens Axboe82f66fb2007-10-22 17:07:37 +020096}
97
Jens Axboe642f1492007-10-24 11:20:47 +020098/**
99 * sg_set_page - Set sg entry to point at given page
100 * @sg: SG entry
101 * @page: The page
102 * @len: Length of data
103 * @offset: Offset into page
104 *
105 * Description:
106 * Use this function to set an sg entry pointing at a page, never assign
107 * the page directly. We encode sg table information in the lower bits
108 * of the page pointer. See sg_page() for looking up the page belonging
109 * to an sg entry.
110 *
111 **/
112static inline void sg_set_page(struct scatterlist *sg, struct page *page,
113 unsigned int len, unsigned int offset)
114{
115 sg_assign_page(sg, page);
116 sg->offset = offset;
117 sg->length = len;
118}
119
Tejun Heo645a8d92007-11-27 09:30:39 +0100120static inline struct page *sg_page(struct scatterlist *sg)
121{
122#ifdef CONFIG_DEBUG_SG
123 BUG_ON(sg->sg_magic != SG_MAGIC);
124 BUG_ON(sg_is_chain(sg));
125#endif
126 return (struct page *)((sg)->page_link & ~0x3);
127}
Jens Axboe82f66fb2007-10-22 17:07:37 +0200128
Jens Axboe18dabf42007-10-22 19:57:20 +0200129/**
130 * sg_set_buf - Set sg entry to point at given data
131 * @sg: SG entry
132 * @buf: Data
133 * @buflen: Data length
134 *
135 **/
Herbert Xu03fd9ce2006-08-14 23:11:53 +1000136static inline void sg_set_buf(struct scatterlist *sg, const void *buf,
Herbert Xud32311f2005-09-17 14:41:40 +1000137 unsigned int buflen)
138{
Rusty Russellac4e97a2013-05-30 09:19:35 +0200139#ifdef CONFIG_DEBUG_SG
140 BUG_ON(!virt_addr_valid(buf));
141#endif
Jens Axboe642f1492007-10-24 11:20:47 +0200142 sg_set_page(sg, virt_to_page(buf), buflen, offset_in_page(buf));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143}
144
Jens Axboe96b418c2007-05-09 09:02:57 +0200145/*
146 * Loop over each sg element, following the pointer to a new list if necessary
147 */
148#define for_each_sg(sglist, sg, nr, __i) \
149 for (__i = 0, sg = (sglist); __i < (nr); __i++, sg = sg_next(sg))
150
Jens Axboe70eb8042007-07-16 21:17:16 +0200151/**
Jens Axboe70eb8042007-07-16 21:17:16 +0200152 * sg_chain - Chain two sglists together
153 * @prv: First scatterlist
154 * @prv_nents: Number of entries in prv
155 * @sgl: Second scatterlist
156 *
Jens Axboe18dabf42007-10-22 19:57:20 +0200157 * Description:
158 * Links @prv@ and @sgl@ together, to form a longer scatterlist.
Jens Axboe70eb8042007-07-16 21:17:16 +0200159 *
Jens Axboe18dabf42007-10-22 19:57:20 +0200160 **/
Jens Axboe70eb8042007-07-16 21:17:16 +0200161static inline void sg_chain(struct scatterlist *prv, unsigned int prv_nents,
162 struct scatterlist *sgl)
163{
Tejun Heo645a8d92007-11-27 09:30:39 +0100164 /*
165 * offset and length are unused for chain entry. Clear them.
166 */
Rusty Russellb801a1e2008-01-11 10:12:55 +0100167 prv[prv_nents - 1].offset = 0;
168 prv[prv_nents - 1].length = 0;
Tejun Heo645a8d92007-11-27 09:30:39 +0100169
Jens Axboe73fd5462007-10-26 09:32:16 +0200170 /*
171 * Set lowest bit to indicate a link pointer, and make sure to clear
172 * the termination bit if it happens to be set.
173 */
174 prv[prv_nents - 1].page_link = ((unsigned long) sgl | 0x01) & ~0x02;
Jens Axboe70eb8042007-07-16 21:17:16 +0200175}
176
Jens Axboe82f66fb2007-10-22 17:07:37 +0200177/**
178 * sg_mark_end - Mark the end of the scatterlist
Jens Axboec46f2332007-10-31 12:06:37 +0100179 * @sg: SG entryScatterlist
Jens Axboe82f66fb2007-10-22 17:07:37 +0200180 *
181 * Description:
Jens Axboec46f2332007-10-31 12:06:37 +0100182 * Marks the passed in sg entry as the termination point for the sg
183 * table. A call to sg_next() on this entry will return NULL.
Jens Axboe82f66fb2007-10-22 17:07:37 +0200184 *
185 **/
Jens Axboec46f2332007-10-31 12:06:37 +0100186static inline void sg_mark_end(struct scatterlist *sg)
Jens Axboe82f66fb2007-10-22 17:07:37 +0200187{
Jens Axboec46f2332007-10-31 12:06:37 +0100188#ifdef CONFIG_DEBUG_SG
189 BUG_ON(sg->sg_magic != SG_MAGIC);
190#endif
191 /*
192 * Set termination bit, clear potential chain bit
193 */
Jens Axboe18dabf42007-10-22 19:57:20 +0200194 sg->page_link |= 0x02;
Jens Axboec46f2332007-10-31 12:06:37 +0100195 sg->page_link &= ~0x01;
Jens Axboe82f66fb2007-10-22 17:07:37 +0200196}
197
Jens Axboe82f66fb2007-10-22 17:07:37 +0200198/**
Paolo Bonzinic8164d82013-03-20 15:37:08 +1030199 * sg_unmark_end - Undo setting the end of the scatterlist
200 * @sg: SG entryScatterlist
201 *
202 * Description:
203 * Removes the termination marker from the given entry of the scatterlist.
204 *
205 **/
206static inline void sg_unmark_end(struct scatterlist *sg)
207{
208#ifdef CONFIG_DEBUG_SG
209 BUG_ON(sg->sg_magic != SG_MAGIC);
210#endif
211 sg->page_link &= ~0x02;
212}
213
214/**
Jens Axboe82f66fb2007-10-22 17:07:37 +0200215 * sg_phys - Return physical address of an sg entry
216 * @sg: SG entry
217 *
218 * Description:
219 * This calls page_to_phys() on the page in this sg entry, and adds the
220 * sg offset. The caller must know that it is legal to call page_to_phys()
221 * on the sg page.
222 *
223 **/
Hugh Dickins85cdffc2007-10-25 09:55:05 +0200224static inline dma_addr_t sg_phys(struct scatterlist *sg)
Jens Axboe82f66fb2007-10-22 17:07:37 +0200225{
226 return page_to_phys(sg_page(sg)) + sg->offset;
227}
228
229/**
230 * sg_virt - Return virtual address of an sg entry
Jens Axboe18dabf42007-10-22 19:57:20 +0200231 * @sg: SG entry
Jens Axboe82f66fb2007-10-22 17:07:37 +0200232 *
233 * Description:
234 * This calls page_address() on the page in this sg entry, and adds the
235 * sg offset. The caller must know that the sg page has a valid virtual
236 * mapping.
237 *
238 **/
239static inline void *sg_virt(struct scatterlist *sg)
240{
241 return page_address(sg_page(sg)) + sg->offset;
242}
243
Maxim Levitsky2e484612012-09-27 12:45:28 +0200244int sg_nents(struct scatterlist *sg);
Tom Lendackycfaed102015-06-01 11:15:25 -0500245int sg_nents_for_len(struct scatterlist *sg, u64 len);
Jens Axboe0db92992007-11-30 09:16:50 +0100246struct scatterlist *sg_next(struct scatterlist *);
247struct scatterlist *sg_last(struct scatterlist *s, unsigned int);
248void sg_init_table(struct scatterlist *, unsigned int);
249void sg_init_one(struct scatterlist *, const void *, unsigned int);
Robert Jarzmikf8bcbe62015-08-08 10:44:10 +0200250int sg_split(struct scatterlist *in, const int in_mapped_nents,
251 const off_t skip, const int nb_splits,
252 const size_t *split_sizes,
253 struct scatterlist **out, int *out_mapped_nents,
254 gfp_t gfp_mask);
Jens Axboe0db92992007-11-30 09:16:50 +0100255
256typedef struct scatterlist *(sg_alloc_fn)(unsigned int, gfp_t);
257typedef void (sg_free_fn)(struct scatterlist *, unsigned int);
258
Christoph Hellwigc53c6d62014-04-15 14:38:31 +0200259void __sg_free_table(struct sg_table *, unsigned int, bool, sg_free_fn *);
Jens Axboe0db92992007-11-30 09:16:50 +0100260void sg_free_table(struct sg_table *);
Christoph Hellwigc53c6d62014-04-15 14:38:31 +0200261int __sg_alloc_table(struct sg_table *, unsigned int, unsigned int,
262 struct scatterlist *, gfp_t, sg_alloc_fn *);
Jens Axboe0db92992007-11-30 09:16:50 +0100263int sg_alloc_table(struct sg_table *, unsigned int, gfp_t);
Tomasz Stanislawskiefc42bc2012-06-18 09:25:01 +0200264int sg_alloc_table_from_pages(struct sg_table *sgt,
265 struct page **pages, unsigned int n_pages,
266 unsigned long offset, unsigned long size,
267 gfp_t gfp_mask);
Jens Axboe0db92992007-11-30 09:16:50 +0100268
Dave Gordon386ecb12015-06-30 14:58:57 -0700269size_t sg_copy_buffer(struct scatterlist *sgl, unsigned int nents, void *buf,
270 size_t buflen, off_t skip, bool to_buffer);
271
FUJITA Tomonorib1adaf62008-03-18 00:15:03 +0900272size_t sg_copy_from_buffer(struct scatterlist *sgl, unsigned int nents,
Dave Gordon2a1bf8f2015-06-30 14:58:54 -0700273 const void *buf, size_t buflen);
FUJITA Tomonorib1adaf62008-03-18 00:15:03 +0900274size_t sg_copy_to_buffer(struct scatterlist *sgl, unsigned int nents,
275 void *buf, size_t buflen);
276
Akinobu Mitadf642ce2013-07-08 16:01:54 -0700277size_t sg_pcopy_from_buffer(struct scatterlist *sgl, unsigned int nents,
Dave Gordon2a1bf8f2015-06-30 14:58:54 -0700278 const void *buf, size_t buflen, off_t skip);
Akinobu Mitadf642ce2013-07-08 16:01:54 -0700279size_t sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents,
280 void *buf, size_t buflen, off_t skip);
281
Jens Axboe0db92992007-11-30 09:16:50 +0100282/*
283 * Maximum number of entries that will be allocated in one piece, if
284 * a list larger than this is required then chaining will be utilized.
285 */
286#define SG_MAX_SINGLE_ALLOC (PAGE_SIZE / sizeof(struct scatterlist))
287
Imre Deaka321e912013-02-27 17:02:56 -0800288/*
Ming Lin9b1d6c82016-04-04 14:48:11 -0700289 * The maximum number of SG segments that we will put inside a
290 * scatterlist (unless chaining is used). Should ideally fit inside a
291 * single page, to avoid a higher order allocation. We could define this
292 * to SG_MAX_SINGLE_ALLOC to pack correctly at the highest order. The
293 * minimum value is 32
294 */
295#define SG_CHUNK_SIZE 128
296
297/*
298 * Like SG_CHUNK_SIZE, but for archs that have sg chaining. This limit
299 * is totally arbitrary, a setting of 2048 will get you at least 8mb ios.
300 */
301#ifdef CONFIG_ARCH_HAS_SG_CHAIN
302#define SG_MAX_SEGMENTS 2048
303#else
304#define SG_MAX_SEGMENTS SG_CHUNK_SIZE
305#endif
306
307#ifdef CONFIG_SG_POOL
308void sg_free_table_chained(struct sg_table *table, bool first_chunk);
309int sg_alloc_table_chained(struct sg_table *table, int nents,
310 struct scatterlist *first_chunk);
311#endif
312
313/*
Imre Deaka321e912013-02-27 17:02:56 -0800314 * sg page iterator
315 *
316 * Iterates over sg entries page-by-page. On each successful iteration,
Imre Deak2db76d72013-03-26 15:14:18 +0200317 * you can call sg_page_iter_page(@piter) and sg_page_iter_dma_address(@piter)
318 * to get the current page and its dma address. @piter->sg will point to the
319 * sg holding this page and @piter->sg_pgoffset to the page's page offset
320 * within the sg. The iteration will stop either when a maximum number of sg
321 * entries was reached or a terminating sg (sg_last(sg) == true) was reached.
Imre Deaka321e912013-02-27 17:02:56 -0800322 */
323struct sg_page_iter {
Imre Deaka321e912013-02-27 17:02:56 -0800324 struct scatterlist *sg; /* sg holding the page */
325 unsigned int sg_pgoffset; /* page offset within the sg */
326
327 /* these are internal states, keep away */
328 unsigned int __nents; /* remaining sg entries */
329 int __pg_advance; /* nr pages to advance at the
330 * next step */
331};
332
333bool __sg_page_iter_next(struct sg_page_iter *piter);
334void __sg_page_iter_start(struct sg_page_iter *piter,
335 struct scatterlist *sglist, unsigned int nents,
336 unsigned long pgoffset);
Imre Deak2db76d72013-03-26 15:14:18 +0200337/**
338 * sg_page_iter_page - get the current page held by the page iterator
339 * @piter: page iterator holding the page
340 */
341static inline struct page *sg_page_iter_page(struct sg_page_iter *piter)
342{
343 return nth_page(sg_page(piter->sg), piter->sg_pgoffset);
344}
345
346/**
347 * sg_page_iter_dma_address - get the dma address of the current page held by
348 * the page iterator.
349 * @piter: page iterator holding the page
350 */
351static inline dma_addr_t sg_page_iter_dma_address(struct sg_page_iter *piter)
352{
353 return sg_dma_address(piter->sg) + (piter->sg_pgoffset << PAGE_SHIFT);
354}
Imre Deaka321e912013-02-27 17:02:56 -0800355
356/**
357 * for_each_sg_page - iterate over the pages of the given sg list
358 * @sglist: sglist to iterate over
359 * @piter: page iterator to hold current page, sg, sg_pgoffset
360 * @nents: maximum number of sg entries to iterate over
361 * @pgoffset: starting page offset
362 */
363#define for_each_sg_page(sglist, piter, nents, pgoffset) \
364 for (__sg_page_iter_start((piter), (sglist), (nents), (pgoffset)); \
365 __sg_page_iter_next(piter);)
Tejun Heo137d3ed2008-07-19 23:03:35 +0900366
367/*
368 * Mapping sg iterator
369 *
370 * Iterates over sg entries mapping page-by-page. On each successful
371 * iteration, @miter->page points to the mapped page and
372 * @miter->length bytes of data can be accessed at @miter->addr. As
373 * long as an interation is enclosed between start and stop, the user
374 * is free to choose control structure and when to stop.
375 *
376 * @miter->consumed is set to @miter->length on each iteration. It
377 * can be adjusted if the user can't consume all the bytes in one go.
378 * Also, a stopped iteration can be resumed by calling next on it.
379 * This is useful when iteration needs to release all resources and
380 * continue later (e.g. at the next interrupt).
381 */
382
383#define SG_MITER_ATOMIC (1 << 0) /* use kmap_atomic */
Sebastian Andrzej Siewior6de7e3562009-06-18 10:19:12 +0200384#define SG_MITER_TO_SG (1 << 1) /* flush back to phys on unmap */
385#define SG_MITER_FROM_SG (1 << 2) /* nop */
Tejun Heo137d3ed2008-07-19 23:03:35 +0900386
387struct sg_mapping_iter {
388 /* the following three fields can be accessed directly */
389 struct page *page; /* currently mapped page */
390 void *addr; /* pointer to the mapped area */
391 size_t length; /* length of the mapped area */
392 size_t consumed; /* number of consumed bytes */
Imre Deak4225fc82013-02-27 17:02:57 -0800393 struct sg_page_iter piter; /* page iterator */
Tejun Heo137d3ed2008-07-19 23:03:35 +0900394
395 /* these are internal states, keep away */
Imre Deak4225fc82013-02-27 17:02:57 -0800396 unsigned int __offset; /* offset within page */
397 unsigned int __remaining; /* remaining bytes on page */
Tejun Heo137d3ed2008-07-19 23:03:35 +0900398 unsigned int __flags;
399};
400
401void sg_miter_start(struct sg_mapping_iter *miter, struct scatterlist *sgl,
402 unsigned int nents, unsigned int flags);
Ming Lei0d6077f2013-11-26 12:43:37 +0800403bool sg_miter_skip(struct sg_mapping_iter *miter, off_t offset);
Tejun Heo137d3ed2008-07-19 23:03:35 +0900404bool sg_miter_next(struct sg_mapping_iter *miter);
405void sg_miter_stop(struct sg_mapping_iter *miter);
406
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407#endif /* _LINUX_SCATTERLIST_H */