scatterlist.h 11.6 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3
#ifndef _LINUX_SCATTERLIST_H
#define _LINUX_SCATTERLIST_H

4
#include <linux/string.h>
C
Christoph Hellwig 已提交
5
#include <linux/types.h>
6 7
#include <linux/bug.h>
#include <linux/mm.h>
J
Jens Axboe 已提交
8 9
#include <asm/io.h>

C
Christoph Hellwig 已提交
10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37
struct scatterlist {
#ifdef CONFIG_DEBUG_SG
	unsigned long	sg_magic;
#endif
	unsigned long	page_link;
	unsigned int	offset;
	unsigned int	length;
	dma_addr_t	dma_address;
#ifdef CONFIG_NEED_SG_DMA_LENGTH
	unsigned int	dma_length;
#endif
};

/*
 * These macros should be used after a dma_map_sg call has been done
 * to get bus addresses of each of the SG entries and their lengths.
 * You should only work with the number of sg entries dma_map_sg
 * returns, or alternatively stop on the first sg_dma_len(sg) which
 * is 0.
 */
#define sg_dma_address(sg)	((sg)->dma_address)

#ifdef CONFIG_NEED_SG_DMA_LENGTH
#define sg_dma_len(sg)		((sg)->dma_length)
#else
#define sg_dma_len(sg)		((sg)->length)
#endif

38 39 40 41 42 43
struct sg_table {
	struct scatterlist *sgl;	/* the list */
	unsigned int nents;		/* number of mapped entries */
	unsigned int orig_nents;	/* original size of list */
};

J
Jens Axboe 已提交
44 45 46
/*
 * Notes on SG table design.
 *
C
Christoph Hellwig 已提交
47 48 49
 * We use the unsigned long page_link field in the scatterlist struct to place
 * the page pointer AND encode information about the sg table as well. The two
 * lower bits are reserved for this information.
J
Jens Axboe 已提交
50 51 52 53 54 55 56 57 58
 *
 * If bit 0 is set, then the page_link contains a pointer to the next sg
 * table list. Otherwise the next entry is at sg + 1.
 *
 * If bit 1 is set, then this sg entry is the last element in a list.
 *
 * See sg_next().
 *
 */
L
Linus Torvalds 已提交
59

J
Jens Axboe 已提交
60 61
#define SG_MAGIC	0x87654321

T
Tejun Heo 已提交
62 63 64 65 66 67 68 69 70 71
/*
 * We overload the LSB of the page pointer to indicate whether it's
 * a valid sg entry, or whether it points to the start of a new scatterlist.
 * Those low bits are there for everyone! (thanks mason :-)
 */
#define sg_is_chain(sg)		((sg)->page_link & 0x01)
#define sg_is_last(sg)		((sg)->page_link & 0x02)
#define sg_chain_ptr(sg)	\
	((struct scatterlist *) ((sg)->page_link & ~0x03))

72
/**
73 74 75
 * sg_assign_page - Assign a given page to an SG entry
 * @sg:		    SG entry
 * @page:	    The page
76 77
 *
 * Description:
78 79
 *   Assign page to sg entry. Also see sg_set_page(), the most commonly used
 *   variant.
80 81
 *
 **/
82
static inline void sg_assign_page(struct scatterlist *sg, struct page *page)
83
{
J
Jens Axboe 已提交
84 85
	unsigned long page_link = sg->page_link & 0x3;

86 87 88 89 90
	/*
	 * In order for the low bit stealing approach to work, pages
	 * must be aligned at a 32-bit boundary as a minimum.
	 */
	BUG_ON((unsigned long) page & 0x03);
J
Jens Axboe 已提交
91 92
#ifdef CONFIG_DEBUG_SG
	BUG_ON(sg->sg_magic != SG_MAGIC);
T
Tejun Heo 已提交
93
	BUG_ON(sg_is_chain(sg));
J
Jens Axboe 已提交
94
#endif
J
Jens Axboe 已提交
95
	sg->page_link = page_link | (unsigned long) page;
96 97
}

98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119
/**
 * sg_set_page - Set sg entry to point at given page
 * @sg:		 SG entry
 * @page:	 The page
 * @len:	 Length of data
 * @offset:	 Offset into page
 *
 * Description:
 *   Use this function to set an sg entry pointing at a page, never assign
 *   the page directly. We encode sg table information in the lower bits
 *   of the page pointer. See sg_page() for looking up the page belonging
 *   to an sg entry.
 *
 **/
static inline void sg_set_page(struct scatterlist *sg, struct page *page,
			       unsigned int len, unsigned int offset)
{
	sg_assign_page(sg, page);
	sg->offset = offset;
	sg->length = len;
}

T
Tejun Heo 已提交
120 121 122 123 124 125 126 127
static inline struct page *sg_page(struct scatterlist *sg)
{
#ifdef CONFIG_DEBUG_SG
	BUG_ON(sg->sg_magic != SG_MAGIC);
	BUG_ON(sg_is_chain(sg));
#endif
	return (struct page *)((sg)->page_link & ~0x3);
}
128

J
Jens Axboe 已提交
129 130 131 132 133 134 135
/**
 * sg_set_buf - Set sg entry to point at given data
 * @sg:		 SG entry
 * @buf:	 Data
 * @buflen:	 Data length
 *
 **/
136
static inline void sg_set_buf(struct scatterlist *sg, const void *buf,
H
Herbert Xu 已提交
137 138
			      unsigned int buflen)
{
139 140 141
#ifdef CONFIG_DEBUG_SG
	BUG_ON(!virt_addr_valid(buf));
#endif
142
	sg_set_page(sg, virt_to_page(buf), buflen, offset_in_page(buf));
L
Linus Torvalds 已提交
143 144
}

145 146 147 148 149 150
/*
 * Loop over each sg element, following the pointer to a new list if necessary
 */
#define for_each_sg(sglist, sg, nr, __i)	\
	for (__i = 0, sg = (sglist); __i < (nr); __i++, sg = sg_next(sg))

151 152 153 154 155 156
/**
 * sg_chain - Chain two sglists together
 * @prv:	First scatterlist
 * @prv_nents:	Number of entries in prv
 * @sgl:	Second scatterlist
 *
J
Jens Axboe 已提交
157 158
 * Description:
 *   Links @prv@ and @sgl@ together, to form a longer scatterlist.
159
 *
J
Jens Axboe 已提交
160
 **/
161 162 163
static inline void sg_chain(struct scatterlist *prv, unsigned int prv_nents,
			    struct scatterlist *sgl)
{
T
Tejun Heo 已提交
164 165 166
	/*
	 * offset and length are unused for chain entry.  Clear them.
	 */
167 168
	prv[prv_nents - 1].offset = 0;
	prv[prv_nents - 1].length = 0;
T
Tejun Heo 已提交
169

170 171 172 173 174
	/*
	 * Set lowest bit to indicate a link pointer, and make sure to clear
	 * the termination bit if it happens to be set.
	 */
	prv[prv_nents - 1].page_link = ((unsigned long) sgl | 0x01) & ~0x02;
175 176
}

177 178
/**
 * sg_mark_end - Mark the end of the scatterlist
J
Jens Axboe 已提交
179
 * @sg:		 SG entryScatterlist
180 181
 *
 * Description:
J
Jens Axboe 已提交
182 183
 *   Marks the passed in sg entry as the termination point for the sg
 *   table. A call to sg_next() on this entry will return NULL.
184 185
 *
 **/
J
Jens Axboe 已提交
186
static inline void sg_mark_end(struct scatterlist *sg)
187
{
J
Jens Axboe 已提交
188 189 190 191 192 193
#ifdef CONFIG_DEBUG_SG
	BUG_ON(sg->sg_magic != SG_MAGIC);
#endif
	/*
	 * Set termination bit, clear potential chain bit
	 */
J
Jens Axboe 已提交
194
	sg->page_link |= 0x02;
J
Jens Axboe 已提交
195
	sg->page_link &= ~0x01;
196 197
}

198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213
/**
 * sg_unmark_end - Undo setting the end of the scatterlist
 * @sg:		 SG entryScatterlist
 *
 * Description:
 *   Removes the termination marker from the given entry of the scatterlist.
 *
 **/
static inline void sg_unmark_end(struct scatterlist *sg)
{
#ifdef CONFIG_DEBUG_SG
	BUG_ON(sg->sg_magic != SG_MAGIC);
#endif
	sg->page_link &= ~0x02;
}

214 215 216 217 218 219 220 221 222 223
/**
 * sg_phys - Return physical address of an sg entry
 * @sg:	     SG entry
 *
 * Description:
 *   This calls page_to_phys() on the page in this sg entry, and adds the
 *   sg offset. The caller must know that it is legal to call page_to_phys()
 *   on the sg page.
 *
 **/
H
Hugh Dickins 已提交
224
static inline dma_addr_t sg_phys(struct scatterlist *sg)
225 226 227 228 229 230
{
	return page_to_phys(sg_page(sg)) + sg->offset;
}

/**
 * sg_virt - Return virtual address of an sg entry
J
Jens Axboe 已提交
231
 * @sg:      SG entry
232 233 234 235 236 237 238 239 240 241 242 243
 *
 * Description:
 *   This calls page_address() on the page in this sg entry, and adds the
 *   sg offset. The caller must know that the sg page has a valid virtual
 *   mapping.
 *
 **/
static inline void *sg_virt(struct scatterlist *sg)
{
	return page_address(sg_page(sg)) + sg->offset;
}

M
Maxim Levitsky 已提交
244
int sg_nents(struct scatterlist *sg);
245
int sg_nents_for_len(struct scatterlist *sg, u64 len);
246 247 248 249
struct scatterlist *sg_next(struct scatterlist *);
struct scatterlist *sg_last(struct scatterlist *s, unsigned int);
void sg_init_table(struct scatterlist *, unsigned int);
void sg_init_one(struct scatterlist *, const void *, unsigned int);
250 251 252 253 254
int sg_split(struct scatterlist *in, const int in_mapped_nents,
	     const off_t skip, const int nb_splits,
	     const size_t *split_sizes,
	     struct scatterlist **out, int *out_mapped_nents,
	     gfp_t gfp_mask);
255 256 257 258

typedef struct scatterlist *(sg_alloc_fn)(unsigned int, gfp_t);
typedef void (sg_free_fn)(struct scatterlist *, unsigned int);

259
void __sg_free_table(struct sg_table *, unsigned int, bool, sg_free_fn *);
260
void sg_free_table(struct sg_table *);
261 262
int __sg_alloc_table(struct sg_table *, unsigned int, unsigned int,
		     struct scatterlist *, gfp_t, sg_alloc_fn *);
263
int sg_alloc_table(struct sg_table *, unsigned int, gfp_t);
264 265 266 267
int sg_alloc_table_from_pages(struct sg_table *sgt,
	struct page **pages, unsigned int n_pages,
	unsigned long offset, unsigned long size,
	gfp_t gfp_mask);
268

269 270 271
size_t sg_copy_buffer(struct scatterlist *sgl, unsigned int nents, void *buf,
		      size_t buflen, off_t skip, bool to_buffer);

272
size_t sg_copy_from_buffer(struct scatterlist *sgl, unsigned int nents,
273
			   const void *buf, size_t buflen);
274 275 276
size_t sg_copy_to_buffer(struct scatterlist *sgl, unsigned int nents,
			 void *buf, size_t buflen);

277
size_t sg_pcopy_from_buffer(struct scatterlist *sgl, unsigned int nents,
278
			    const void *buf, size_t buflen, off_t skip);
279 280 281
size_t sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents,
			  void *buf, size_t buflen, off_t skip);

282 283 284 285 286 287
/*
 * Maximum number of entries that will be allocated in one piece, if
 * a list larger than this is required then chaining will be utilized.
 */
#define SG_MAX_SINGLE_ALLOC		(PAGE_SIZE / sizeof(struct scatterlist))

288 289 290 291
/*
 * sg page iterator
 *
 * Iterates over sg entries page-by-page.  On each successful iteration,
292 293 294 295 296
 * you can call sg_page_iter_page(@piter) and sg_page_iter_dma_address(@piter)
 * to get the current page and its dma address. @piter->sg will point to the
 * sg holding this page and @piter->sg_pgoffset to the page's page offset
 * within the sg. The iteration will stop either when a maximum number of sg
 * entries was reached or a terminating sg (sg_last(sg) == true) was reached.
297 298 299 300 301 302 303 304 305 306 307 308 309 310 311
 */
struct sg_page_iter {
	struct scatterlist	*sg;		/* sg holding the page */
	unsigned int		sg_pgoffset;	/* page offset within the sg */

	/* these are internal states, keep away */
	unsigned int		__nents;	/* remaining sg entries */
	int			__pg_advance;	/* nr pages to advance at the
						 * next step */
};

bool __sg_page_iter_next(struct sg_page_iter *piter);
void __sg_page_iter_start(struct sg_page_iter *piter,
			  struct scatterlist *sglist, unsigned int nents,
			  unsigned long pgoffset);
312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329
/**
 * sg_page_iter_page - get the current page held by the page iterator
 * @piter:	page iterator holding the page
 */
static inline struct page *sg_page_iter_page(struct sg_page_iter *piter)
{
	return nth_page(sg_page(piter->sg), piter->sg_pgoffset);
}

/**
 * sg_page_iter_dma_address - get the dma address of the current page held by
 * the page iterator.
 * @piter:	page iterator holding the page
 */
static inline dma_addr_t sg_page_iter_dma_address(struct sg_page_iter *piter)
{
	return sg_dma_address(piter->sg) + (piter->sg_pgoffset << PAGE_SHIFT);
}
330 331 332 333 334 335 336 337 338 339 340

/**
 * for_each_sg_page - iterate over the pages of the given sg list
 * @sglist:	sglist to iterate over
 * @piter:	page iterator to hold current page, sg, sg_pgoffset
 * @nents:	maximum number of sg entries to iterate over
 * @pgoffset:	starting page offset
 */
#define for_each_sg_page(sglist, piter, nents, pgoffset)		   \
	for (__sg_page_iter_start((piter), (sglist), (nents), (pgoffset)); \
	     __sg_page_iter_next(piter);)
T
Tejun Heo 已提交
341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358

/*
 * Mapping sg iterator
 *
 * Iterates over sg entries mapping page-by-page.  On each successful
 * iteration, @miter->page points to the mapped page and
 * @miter->length bytes of data can be accessed at @miter->addr.  As
 * long as an interation is enclosed between start and stop, the user
 * is free to choose control structure and when to stop.
 *
 * @miter->consumed is set to @miter->length on each iteration.  It
 * can be adjusted if the user can't consume all the bytes in one go.
 * Also, a stopped iteration can be resumed by calling next on it.
 * This is useful when iteration needs to release all resources and
 * continue later (e.g. at the next interrupt).
 */

#define SG_MITER_ATOMIC		(1 << 0)	 /* use kmap_atomic */
359 360
#define SG_MITER_TO_SG		(1 << 1)	/* flush back to phys on unmap */
#define SG_MITER_FROM_SG	(1 << 2)	/* nop */
T
Tejun Heo 已提交
361 362 363 364 365 366 367

struct sg_mapping_iter {
	/* the following three fields can be accessed directly */
	struct page		*page;		/* currently mapped page */
	void			*addr;		/* pointer to the mapped area */
	size_t			length;		/* length of the mapped area */
	size_t			consumed;	/* number of consumed bytes */
368
	struct sg_page_iter	piter;		/* page iterator */
T
Tejun Heo 已提交
369 370

	/* these are internal states, keep away */
371 372
	unsigned int		__offset;	/* offset within page */
	unsigned int		__remaining;	/* remaining bytes on page */
T
Tejun Heo 已提交
373 374 375 376 377
	unsigned int		__flags;
};

void sg_miter_start(struct sg_mapping_iter *miter, struct scatterlist *sgl,
		    unsigned int nents, unsigned int flags);
M
Ming Lei 已提交
378
bool sg_miter_skip(struct sg_mapping_iter *miter, off_t offset);
T
Tejun Heo 已提交
379 380 381
bool sg_miter_next(struct sg_mapping_iter *miter);
void sg_miter_stop(struct sg_mapping_iter *miter);

L
Linus Torvalds 已提交
382
#endif /* _LINUX_SCATTERLIST_H */