blob: ad419757241ffc7c5d5346f4b82462f5a83f1897 [file] [log] [blame]
Chris Leechc13c8262006-05-23 17:18:44 -07001/*
2 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the Free
6 * Software Foundation; either version 2 of the License, or (at your option)
7 * any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
Chris Leechc13c8262006-05-23 17:18:44 -070014 * The full GNU General Public License is included in this distribution in the
15 * file called COPYING.
16 */
Russell King - ARM Linuxd2ebfb32012-03-06 22:34:26 +000017#ifndef LINUX_DMAENGINE_H
18#define LINUX_DMAENGINE_H
David Woodhouse1c0f16e2006-06-27 02:53:56 -070019
Chris Leechc13c8262006-05-23 17:18:44 -070020#include <linux/device.h>
Stephen Warren0ad7c002013-11-26 10:04:22 -070021#include <linux/err.h>
Chris Leechc13c8262006-05-23 17:18:44 -070022#include <linux/uio.h>
Paul Gortmaker187f1882011-11-23 20:12:59 -050023#include <linux/bug.h>
Vinod Koul90b44f82011-07-25 19:57:52 +053024#include <linux/scatterlist.h>
Paul Gortmakera8efa9d2011-07-29 16:55:11 +100025#include <linux/bitmap.h>
Viresh Kumardcc043d2012-02-01 16:12:18 +053026#include <linux/types.h>
Paul Gortmakera8efa9d2011-07-29 16:55:11 +100027#include <asm/page.h>
Alexey Dobriyanb7f080c2011-06-16 11:01:34 +000028
Chris Leechc13c8262006-05-23 17:18:44 -070029/**
Randy Dunlapfe4ada22006-07-03 19:44:51 -070030 * typedef dma_cookie_t - an opaque DMA cookie
Chris Leechc13c8262006-05-23 17:18:44 -070031 *
32 * if dma_cookie_t is >0 it's a DMA request cookie, <0 it's an error code
33 */
34typedef s32 dma_cookie_t;
Steven J. Magnani76bd0612010-02-28 22:18:16 -070035#define DMA_MIN_COOKIE 1
Chris Leechc13c8262006-05-23 17:18:44 -070036
Dan Carpenter71ea1482013-08-10 10:46:50 +030037static inline int dma_submit_error(dma_cookie_t cookie)
38{
39 return cookie < 0 ? cookie : 0;
40}
Chris Leechc13c8262006-05-23 17:18:44 -070041
42/**
43 * enum dma_status - DMA transaction status
Vinod Kouladfedd92013-10-16 13:29:02 +053044 * @DMA_COMPLETE: transaction completed
Chris Leechc13c8262006-05-23 17:18:44 -070045 * @DMA_IN_PROGRESS: transaction not yet processed
Linus Walleij07934482010-03-26 16:50:49 -070046 * @DMA_PAUSED: transaction is paused
Chris Leechc13c8262006-05-23 17:18:44 -070047 * @DMA_ERROR: transaction failed
48 */
49enum dma_status {
Vinod Koul7db5f722013-10-17 07:29:57 +053050 DMA_COMPLETE,
Chris Leechc13c8262006-05-23 17:18:44 -070051 DMA_IN_PROGRESS,
Linus Walleij07934482010-03-26 16:50:49 -070052 DMA_PAUSED,
Chris Leechc13c8262006-05-23 17:18:44 -070053 DMA_ERROR,
54};
55
56/**
Dan Williams7405f742007-01-02 11:10:43 -070057 * enum dma_transaction_type - DMA transaction types/indexes
Dan Williams138f4c32009-09-08 17:42:51 -070058 *
59 * Note: The DMA_ASYNC_TX capability is not to be set by drivers. It is
60 * automatically set as dma devices are registered.
Dan Williams7405f742007-01-02 11:10:43 -070061 */
62enum dma_transaction_type {
63 DMA_MEMCPY,
64 DMA_XOR,
Dan Williamsb2f46fd2009-07-14 12:20:36 -070065 DMA_PQ,
Dan Williams099f53c2009-04-08 14:28:37 -070066 DMA_XOR_VAL,
67 DMA_PQ_VAL,
Dan Williams7405f742007-01-02 11:10:43 -070068 DMA_INTERRUPT,
Ira Snydera86ee032010-09-30 11:46:44 +000069 DMA_SG,
Dan Williams59b5ec22009-01-06 11:38:15 -070070 DMA_PRIVATE,
Dan Williams138f4c32009-09-08 17:42:51 -070071 DMA_ASYNC_TX,
Haavard Skinnemoendc0ee6432008-07-08 11:59:35 -070072 DMA_SLAVE,
Sascha Hauer782bc952010-09-30 13:56:32 +000073 DMA_CYCLIC,
Jassi Brarb14dab72011-10-13 12:33:30 +053074 DMA_INTERLEAVE,
Dan Williams7405f742007-01-02 11:10:43 -070075/* last transaction type for creation of the capabilities mask */
Jassi Brarb14dab72011-10-13 12:33:30 +053076 DMA_TX_TYPE_END,
77};
Haavard Skinnemoendc0ee6432008-07-08 11:59:35 -070078
Vinod Koul49920bc2011-10-13 15:15:27 +053079/**
80 * enum dma_transfer_direction - dma transfer mode and direction indicator
81 * @DMA_MEM_TO_MEM: Async/Memcpy mode
82 * @DMA_MEM_TO_DEV: Slave mode & From Memory to Device
83 * @DMA_DEV_TO_MEM: Slave mode & From Device to Memory
84 * @DMA_DEV_TO_DEV: Slave mode & From Device to Device
85 */
86enum dma_transfer_direction {
87 DMA_MEM_TO_MEM,
88 DMA_MEM_TO_DEV,
89 DMA_DEV_TO_MEM,
90 DMA_DEV_TO_DEV,
Shawn Guo62268ce2011-12-13 23:48:03 +080091 DMA_TRANS_NONE,
Vinod Koul49920bc2011-10-13 15:15:27 +053092};
Dan Williams7405f742007-01-02 11:10:43 -070093
94/**
Jassi Brarb14dab72011-10-13 12:33:30 +053095 * Interleaved Transfer Request
96 * ----------------------------
97 * A chunk is collection of contiguous bytes to be transfered.
98 * The gap(in bytes) between two chunks is called inter-chunk-gap(ICG).
99 * ICGs may or maynot change between chunks.
100 * A FRAME is the smallest series of contiguous {chunk,icg} pairs,
101 * that when repeated an integral number of times, specifies the transfer.
102 * A transfer template is specification of a Frame, the number of times
103 * it is to be repeated and other per-transfer attributes.
104 *
105 * Practically, a client driver would have ready a template for each
106 * type of transfer it is going to need during its lifetime and
107 * set only 'src_start' and 'dst_start' before submitting the requests.
108 *
109 *
110 * | Frame-1 | Frame-2 | ~ | Frame-'numf' |
111 * |====....==.===...=...|====....==.===...=...| ~ |====....==.===...=...|
112 *
113 * == Chunk size
114 * ... ICG
115 */
116
117/**
118 * struct data_chunk - Element of scatter-gather list that makes a frame.
119 * @size: Number of bytes to read from source.
120 * size_dst := fn(op, size_src), so doesn't mean much for destination.
121 * @icg: Number of bytes to jump after last src/dst address of this
122 * chunk and before first src/dst address for next chunk.
123 * Ignored for dst(assumed 0), if dst_inc is true and dst_sgl is false.
124 * Ignored for src(assumed 0), if src_inc is true and src_sgl is false.
125 */
126struct data_chunk {
127 size_t size;
128 size_t icg;
129};
130
131/**
132 * struct dma_interleaved_template - Template to convey DMAC the transfer pattern
133 * and attributes.
134 * @src_start: Bus address of source for the first chunk.
135 * @dst_start: Bus address of destination for the first chunk.
136 * @dir: Specifies the type of Source and Destination.
137 * @src_inc: If the source address increments after reading from it.
138 * @dst_inc: If the destination address increments after writing to it.
139 * @src_sgl: If the 'icg' of sgl[] applies to Source (scattered read).
140 * Otherwise, source is read contiguously (icg ignored).
141 * Ignored if src_inc is false.
142 * @dst_sgl: If the 'icg' of sgl[] applies to Destination (scattered write).
143 * Otherwise, destination is filled contiguously (icg ignored).
144 * Ignored if dst_inc is false.
145 * @numf: Number of frames in this template.
146 * @frame_size: Number of chunks in a frame i.e, size of sgl[].
147 * @sgl: Array of {chunk,icg} pairs that make up a frame.
148 */
149struct dma_interleaved_template {
150 dma_addr_t src_start;
151 dma_addr_t dst_start;
152 enum dma_transfer_direction dir;
153 bool src_inc;
154 bool dst_inc;
155 bool src_sgl;
156 bool dst_sgl;
157 size_t numf;
158 size_t frame_size;
159 struct data_chunk sgl[0];
160};
161
162/**
Dan Williams636bdea2008-04-17 20:17:26 -0700163 * enum dma_ctrl_flags - DMA flags to augment operation preparation,
Dan Williamsb2f46fd2009-07-14 12:20:36 -0700164 * control completion, and communicate status.
Dan Williamsd4c56f92008-02-02 19:49:58 -0700165 * @DMA_PREP_INTERRUPT - trigger an interrupt (callback) upon completion of
Dan Williamsb2f46fd2009-07-14 12:20:36 -0700166 * this transaction
Guennadi Liakhovetskia88f6662009-12-10 18:35:15 +0100167 * @DMA_CTRL_ACK - if clear, the descriptor cannot be reused until the client
Dan Williamsb2f46fd2009-07-14 12:20:36 -0700168 * acknowledges receipt, i.e. has has a chance to establish any dependency
169 * chains
Dan Williamsb2f46fd2009-07-14 12:20:36 -0700170 * @DMA_PREP_PQ_DISABLE_P - prevent generation of P while generating Q
171 * @DMA_PREP_PQ_DISABLE_Q - prevent generation of Q while generating P
172 * @DMA_PREP_CONTINUE - indicate to a driver that it is reusing buffers as
173 * sources that were the result of a previous operation, in the case of a PQ
174 * operation it continues the calculation with new sources
Dan Williams0403e382009-09-08 17:42:50 -0700175 * @DMA_PREP_FENCE - tell the driver that subsequent operations depend
176 * on the result of this operation
Dan Williamsd4c56f92008-02-02 19:49:58 -0700177 */
Dan Williams636bdea2008-04-17 20:17:26 -0700178enum dma_ctrl_flags {
Dan Williamsd4c56f92008-02-02 19:49:58 -0700179 DMA_PREP_INTERRUPT = (1 << 0),
Dan Williams636bdea2008-04-17 20:17:26 -0700180 DMA_CTRL_ACK = (1 << 1),
Bartlomiej Zolnierkiewicz0776ae72013-10-18 19:35:33 +0200181 DMA_PREP_PQ_DISABLE_P = (1 << 2),
182 DMA_PREP_PQ_DISABLE_Q = (1 << 3),
183 DMA_PREP_CONTINUE = (1 << 4),
184 DMA_PREP_FENCE = (1 << 5),
Dan Williamsd4c56f92008-02-02 19:49:58 -0700185};
186
187/**
Dan Williamsad283ea2009-08-29 19:09:26 -0700188 * enum sum_check_bits - bit position of pq_check_flags
189 */
190enum sum_check_bits {
191 SUM_CHECK_P = 0,
192 SUM_CHECK_Q = 1,
193};
194
195/**
196 * enum pq_check_flags - result of async_{xor,pq}_zero_sum operations
197 * @SUM_CHECK_P_RESULT - 1 if xor zero sum error, 0 otherwise
198 * @SUM_CHECK_Q_RESULT - 1 if reed-solomon zero sum error, 0 otherwise
199 */
200enum sum_check_flags {
201 SUM_CHECK_P_RESULT = (1 << SUM_CHECK_P),
202 SUM_CHECK_Q_RESULT = (1 << SUM_CHECK_Q),
203};
204
205
206/**
Dan Williams7405f742007-01-02 11:10:43 -0700207 * dma_cap_mask_t - capabilities bitmap modeled after cpumask_t.
208 * See linux/cpumask.h
209 */
210typedef struct { DECLARE_BITMAP(bits, DMA_TX_TYPE_END); } dma_cap_mask_t;
211
212/**
Chris Leechc13c8262006-05-23 17:18:44 -0700213 * struct dma_chan_percpu - the per-CPU part of struct dma_chan
Chris Leechc13c8262006-05-23 17:18:44 -0700214 * @memcpy_count: transaction counter
215 * @bytes_transferred: byte counter
216 */
217
218struct dma_chan_percpu {
Chris Leechc13c8262006-05-23 17:18:44 -0700219 /* stats */
220 unsigned long memcpy_count;
221 unsigned long bytes_transferred;
222};
223
224/**
225 * struct dma_chan - devices supply DMA channels, clients use them
Randy Dunlapfe4ada22006-07-03 19:44:51 -0700226 * @device: ptr to the dma device who supplies this channel, always !%NULL
Chris Leechc13c8262006-05-23 17:18:44 -0700227 * @cookie: last cookie value returned to client
Russell King - ARM Linux4d4e58d2012-03-06 22:34:06 +0000228 * @completed_cookie: last completed cookie for this channel
Randy Dunlapfe4ada22006-07-03 19:44:51 -0700229 * @chan_id: channel ID for sysfs
Dan Williams41d5e592009-01-06 11:38:21 -0700230 * @dev: class device for sysfs
Chris Leechc13c8262006-05-23 17:18:44 -0700231 * @device_node: used to add this to the device chan list
232 * @local: per-cpu pointer to a struct dma_chan_percpu
Vinod Koul868d2ee2013-12-18 21:39:39 +0530233 * @client_count: how many clients are using this channel
Dan Williamsbec08512009-01-06 11:38:14 -0700234 * @table_count: number of appearances in the mem-to-mem allocation table
Dan Williams287d8592009-02-18 14:48:26 -0800235 * @private: private data for certain client-channel associations
Chris Leechc13c8262006-05-23 17:18:44 -0700236 */
237struct dma_chan {
Chris Leechc13c8262006-05-23 17:18:44 -0700238 struct dma_device *device;
239 dma_cookie_t cookie;
Russell King - ARM Linux4d4e58d2012-03-06 22:34:06 +0000240 dma_cookie_t completed_cookie;
Chris Leechc13c8262006-05-23 17:18:44 -0700241
242 /* sysfs */
243 int chan_id;
Dan Williams41d5e592009-01-06 11:38:21 -0700244 struct dma_chan_dev *dev;
Chris Leechc13c8262006-05-23 17:18:44 -0700245
Chris Leechc13c8262006-05-23 17:18:44 -0700246 struct list_head device_node;
Tejun Heoa29d8b82010-02-02 14:39:15 +0900247 struct dma_chan_percpu __percpu *local;
Dan Williams7cc5bf92008-07-08 11:58:21 -0700248 int client_count;
Dan Williamsbec08512009-01-06 11:38:14 -0700249 int table_count;
Dan Williams287d8592009-02-18 14:48:26 -0800250 void *private;
Chris Leechc13c8262006-05-23 17:18:44 -0700251};
252
Dan Williams41d5e592009-01-06 11:38:21 -0700253/**
254 * struct dma_chan_dev - relate sysfs device node to backing channel device
Vinod Koul868d2ee2013-12-18 21:39:39 +0530255 * @chan: driver channel device
256 * @device: sysfs device
257 * @dev_id: parent dma_device dev_id
258 * @idr_ref: reference count to gate release of dma_device dev_id
Dan Williams41d5e592009-01-06 11:38:21 -0700259 */
260struct dma_chan_dev {
261 struct dma_chan *chan;
262 struct device device;
Dan Williams864498a2009-01-06 11:38:21 -0700263 int dev_id;
264 atomic_t *idr_ref;
Dan Williams41d5e592009-01-06 11:38:21 -0700265};
266
Linus Walleijc156d0a2010-08-04 13:37:33 +0200267/**
Alexander Popovba730342014-05-15 18:15:31 +0400268 * enum dma_slave_buswidth - defines bus width of the DMA slave
Linus Walleijc156d0a2010-08-04 13:37:33 +0200269 * device, source or target buses
270 */
271enum dma_slave_buswidth {
272 DMA_SLAVE_BUSWIDTH_UNDEFINED = 0,
273 DMA_SLAVE_BUSWIDTH_1_BYTE = 1,
274 DMA_SLAVE_BUSWIDTH_2_BYTES = 2,
Peter Ujfalusi93c6ee92014-07-03 07:51:52 +0300275 DMA_SLAVE_BUSWIDTH_3_BYTES = 3,
Linus Walleijc156d0a2010-08-04 13:37:33 +0200276 DMA_SLAVE_BUSWIDTH_4_BYTES = 4,
277 DMA_SLAVE_BUSWIDTH_8_BYTES = 8,
Laurent Pinchart534a7292014-08-06 10:52:41 +0200278 DMA_SLAVE_BUSWIDTH_16_BYTES = 16,
279 DMA_SLAVE_BUSWIDTH_32_BYTES = 32,
280 DMA_SLAVE_BUSWIDTH_64_BYTES = 64,
Linus Walleijc156d0a2010-08-04 13:37:33 +0200281};
282
283/**
284 * struct dma_slave_config - dma slave channel runtime config
285 * @direction: whether the data shall go in or out on this slave
Alexander Popov397321f2013-12-16 12:12:17 +0400286 * channel, right now. DMA_MEM_TO_DEV and DMA_DEV_TO_MEM are
Laurent Pinchartd9ff9582014-08-20 19:20:53 +0200287 * legal values. DEPRECATED, drivers should use the direction argument
288 * to the device_prep_slave_sg and device_prep_dma_cyclic functions or
289 * the dir field in the dma_interleaved_template structure.
Linus Walleijc156d0a2010-08-04 13:37:33 +0200290 * @src_addr: this is the physical address where DMA slave data
291 * should be read (RX), if the source is memory this argument is
292 * ignored.
293 * @dst_addr: this is the physical address where DMA slave data
294 * should be written (TX), if the source is memory this argument
295 * is ignored.
296 * @src_addr_width: this is the width in bytes of the source (RX)
297 * register where DMA data shall be read. If the source
298 * is memory this may be ignored depending on architecture.
299 * Legal values: 1, 2, 4, 8.
300 * @dst_addr_width: same as src_addr_width but for destination
301 * target (TX) mutatis mutandis.
302 * @src_maxburst: the maximum number of words (note: words, as in
303 * units of the src_addr_width member, not bytes) that can be sent
304 * in one burst to the device. Typically something like half the
305 * FIFO depth on I/O peripherals so you don't overflow it. This
306 * may or may not be applicable on memory sources.
307 * @dst_maxburst: same as src_maxburst but for destination target
308 * mutatis mutandis.
Viresh Kumardcc043d2012-02-01 16:12:18 +0530309 * @device_fc: Flow Controller Settings. Only valid for slave channels. Fill
310 * with 'true' if peripheral should be flow controller. Direction will be
311 * selected at Runtime.
Laxman Dewangan4fd1e322012-06-06 10:55:26 +0530312 * @slave_id: Slave requester id. Only valid for slave channels. The dma
313 * slave peripheral will have unique id as dma requester which need to be
314 * pass as slave config.
Linus Walleijc156d0a2010-08-04 13:37:33 +0200315 *
316 * This struct is passed in as configuration data to a DMA engine
317 * in order to set up a certain channel for DMA transport at runtime.
318 * The DMA device/engine has to provide support for an additional
Maxime Ripard2c44ad92014-11-17 14:42:54 +0100319 * callback in the dma_device structure, device_config and this struct
320 * will then be passed in as an argument to the function.
Linus Walleijc156d0a2010-08-04 13:37:33 +0200321 *
Lars-Peter Clausen7cbccb52014-02-16 14:21:22 +0100322 * The rationale for adding configuration information to this struct is as
323 * follows: if it is likely that more than one DMA slave controllers in
324 * the world will support the configuration option, then make it generic.
325 * If not: if it is fixed so that it be sent in static from the platform
326 * data, then prefer to do that.
Linus Walleijc156d0a2010-08-04 13:37:33 +0200327 */
328struct dma_slave_config {
Vinod Koul49920bc2011-10-13 15:15:27 +0530329 enum dma_transfer_direction direction;
Linus Walleijc156d0a2010-08-04 13:37:33 +0200330 dma_addr_t src_addr;
331 dma_addr_t dst_addr;
332 enum dma_slave_buswidth src_addr_width;
333 enum dma_slave_buswidth dst_addr_width;
334 u32 src_maxburst;
335 u32 dst_maxburst;
Viresh Kumardcc043d2012-02-01 16:12:18 +0530336 bool device_fc;
Laxman Dewangan4fd1e322012-06-06 10:55:26 +0530337 unsigned int slave_id;
Linus Walleijc156d0a2010-08-04 13:37:33 +0200338};
339
Lars-Peter Clausen50720562014-01-11 14:02:16 +0100340/**
341 * enum dma_residue_granularity - Granularity of the reported transfer residue
342 * @DMA_RESIDUE_GRANULARITY_DESCRIPTOR: Residue reporting is not support. The
343 * DMA channel is only able to tell whether a descriptor has been completed or
344 * not, which means residue reporting is not supported by this channel. The
345 * residue field of the dma_tx_state field will always be 0.
346 * @DMA_RESIDUE_GRANULARITY_SEGMENT: Residue is updated after each successfully
347 * completed segment of the transfer (For cyclic transfers this is after each
348 * period). This is typically implemented by having the hardware generate an
349 * interrupt after each transferred segment and then the drivers updates the
350 * outstanding residue by the size of the segment. Another possibility is if
351 * the hardware supports scatter-gather and the segment descriptor has a field
352 * which gets set after the segment has been completed. The driver then counts
353 * the number of segments without the flag set to compute the residue.
354 * @DMA_RESIDUE_GRANULARITY_BURST: Residue is updated after each transferred
355 * burst. This is typically only supported if the hardware has a progress
356 * register of some sort (E.g. a register with the current read/write address
357 * or a register with the amount of bursts/beats/bytes that have been
358 * transferred or still need to be transferred).
359 */
360enum dma_residue_granularity {
361 DMA_RESIDUE_GRANULARITY_DESCRIPTOR = 0,
362 DMA_RESIDUE_GRANULARITY_SEGMENT = 1,
363 DMA_RESIDUE_GRANULARITY_BURST = 2,
364};
365
Vinod Koul221a27c72013-07-08 14:15:25 +0530366/* struct dma_slave_caps - expose capabilities of a slave channel only
367 *
368 * @src_addr_widths: bit mask of src addr widths the channel supports
Maxime Ripardceacbdb2014-11-17 14:41:57 +0100369 * @dst_addr_widths: bit mask of dstn addr widths the channel supports
Vinod Koul221a27c72013-07-08 14:15:25 +0530370 * @directions: bit mask of slave direction the channel supported
371 * since the enum dma_transfer_direction is not defined as bits for each
372 * type of direction, the dma controller should fill (1 << <TYPE>) and same
373 * should be checked by controller as well
374 * @cmd_pause: true, if pause and thereby resume is supported
375 * @cmd_terminate: true, if terminate cmd is supported
Lars-Peter Clausen50720562014-01-11 14:02:16 +0100376 * @residue_granularity: granularity of the reported transfer residue
Vinod Koul221a27c72013-07-08 14:15:25 +0530377 */
378struct dma_slave_caps {
379 u32 src_addr_widths;
Maxime Ripardceacbdb2014-11-17 14:41:57 +0100380 u32 dst_addr_widths;
Vinod Koul221a27c72013-07-08 14:15:25 +0530381 u32 directions;
382 bool cmd_pause;
383 bool cmd_terminate;
Lars-Peter Clausen50720562014-01-11 14:02:16 +0100384 enum dma_residue_granularity residue_granularity;
Vinod Koul221a27c72013-07-08 14:15:25 +0530385};
386
Dan Williams41d5e592009-01-06 11:38:21 -0700387static inline const char *dma_chan_name(struct dma_chan *chan)
388{
389 return dev_name(&chan->dev->device);
390}
Dan Williamsd379b012007-07-09 11:56:42 -0700391
Chris Leechc13c8262006-05-23 17:18:44 -0700392void dma_chan_cleanup(struct kref *kref);
393
Chris Leechc13c8262006-05-23 17:18:44 -0700394/**
Dan Williams59b5ec22009-01-06 11:38:15 -0700395 * typedef dma_filter_fn - callback filter for dma_request_channel
396 * @chan: channel to be reviewed
397 * @filter_param: opaque parameter passed through dma_request_channel
398 *
399 * When this optional parameter is specified in a call to dma_request_channel a
400 * suitable channel is passed to this routine for further dispositioning before
401 * being returned. Where 'suitable' indicates a non-busy channel that
Dan Williams7dd60252009-01-06 11:38:19 -0700402 * satisfies the given capability mask. It returns 'true' to indicate that the
403 * channel is suitable.
Dan Williams59b5ec22009-01-06 11:38:15 -0700404 */
Dan Williams7dd60252009-01-06 11:38:19 -0700405typedef bool (*dma_filter_fn)(struct dma_chan *chan, void *filter_param);
Dan Williams59b5ec22009-01-06 11:38:15 -0700406
Dan Williams7405f742007-01-02 11:10:43 -0700407typedef void (*dma_async_tx_callback)(void *dma_async_param);
Dan Williamsd38a8c62013-10-18 19:35:23 +0200408
409struct dmaengine_unmap_data {
Xuelin Shic1f43dd2014-05-21 14:02:37 -0700410 u8 map_cnt;
Dan Williamsd38a8c62013-10-18 19:35:23 +0200411 u8 to_cnt;
412 u8 from_cnt;
413 u8 bidi_cnt;
414 struct device *dev;
415 struct kref kref;
416 size_t len;
417 dma_addr_t addr[0];
418};
419
Dan Williams7405f742007-01-02 11:10:43 -0700420/**
421 * struct dma_async_tx_descriptor - async transaction descriptor
422 * ---dma generic offload fields---
423 * @cookie: tracking cookie for this transaction, set to -EBUSY if
424 * this tx is sitting on a dependency list
Dan Williams636bdea2008-04-17 20:17:26 -0700425 * @flags: flags to augment operation preparation, control completion, and
426 * communicate status
Dan Williams7405f742007-01-02 11:10:43 -0700427 * @phys: physical address of the descriptor
Dan Williams7405f742007-01-02 11:10:43 -0700428 * @chan: target channel for this operation
Vinod Koulaba96ba2014-12-05 20:49:07 +0530429 * @tx_submit: accept the descriptor, assign ordered cookie and mark the
430 * descriptor pending. To be pushed on .issue_pending() call
Dan Williams7405f742007-01-02 11:10:43 -0700431 * @callback: routine to call after this operation is complete
432 * @callback_param: general parameter to pass to the callback routine
433 * ---async_tx api specific fields---
Dan Williams19242d72008-04-17 20:17:25 -0700434 * @next: at completion submit this descriptor
Dan Williams7405f742007-01-02 11:10:43 -0700435 * @parent: pointer to the next level up in the dependency chain
Dan Williams19242d72008-04-17 20:17:25 -0700436 * @lock: protect the parent and next pointers
Dan Williams7405f742007-01-02 11:10:43 -0700437 */
438struct dma_async_tx_descriptor {
439 dma_cookie_t cookie;
Dan Williams636bdea2008-04-17 20:17:26 -0700440 enum dma_ctrl_flags flags; /* not a 'long' to pack with cookie */
Dan Williams7405f742007-01-02 11:10:43 -0700441 dma_addr_t phys;
Dan Williams7405f742007-01-02 11:10:43 -0700442 struct dma_chan *chan;
443 dma_cookie_t (*tx_submit)(struct dma_async_tx_descriptor *tx);
Dan Williams7405f742007-01-02 11:10:43 -0700444 dma_async_tx_callback callback;
445 void *callback_param;
Dan Williamsd38a8c62013-10-18 19:35:23 +0200446 struct dmaengine_unmap_data *unmap;
Dan Williams5fc6d892010-10-07 16:44:50 -0700447#ifdef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH
Dan Williams19242d72008-04-17 20:17:25 -0700448 struct dma_async_tx_descriptor *next;
Dan Williams7405f742007-01-02 11:10:43 -0700449 struct dma_async_tx_descriptor *parent;
450 spinlock_t lock;
Dan Williamscaa20d972010-05-17 16:24:16 -0700451#endif
Dan Williams7405f742007-01-02 11:10:43 -0700452};
453
Dan Williams89716462013-10-18 19:35:25 +0200454#ifdef CONFIG_DMA_ENGINE
Dan Williamsd38a8c62013-10-18 19:35:23 +0200455static inline void dma_set_unmap(struct dma_async_tx_descriptor *tx,
456 struct dmaengine_unmap_data *unmap)
457{
458 kref_get(&unmap->kref);
459 tx->unmap = unmap;
460}
461
Dan Williams89716462013-10-18 19:35:25 +0200462struct dmaengine_unmap_data *
463dmaengine_get_unmap_data(struct device *dev, int nr, gfp_t flags);
Dan Williams45c463a2013-10-18 19:35:24 +0200464void dmaengine_unmap_put(struct dmaengine_unmap_data *unmap);
Dan Williams89716462013-10-18 19:35:25 +0200465#else
466static inline void dma_set_unmap(struct dma_async_tx_descriptor *tx,
467 struct dmaengine_unmap_data *unmap)
468{
469}
470static inline struct dmaengine_unmap_data *
471dmaengine_get_unmap_data(struct device *dev, int nr, gfp_t flags)
472{
473 return NULL;
474}
475static inline void dmaengine_unmap_put(struct dmaengine_unmap_data *unmap)
476{
477}
478#endif
Dan Williams45c463a2013-10-18 19:35:24 +0200479
Dan Williamsd38a8c62013-10-18 19:35:23 +0200480static inline void dma_descriptor_unmap(struct dma_async_tx_descriptor *tx)
481{
482 if (tx->unmap) {
Dan Williams45c463a2013-10-18 19:35:24 +0200483 dmaengine_unmap_put(tx->unmap);
Dan Williamsd38a8c62013-10-18 19:35:23 +0200484 tx->unmap = NULL;
485 }
486}
487
Dan Williams5fc6d892010-10-07 16:44:50 -0700488#ifndef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH
Dan Williamscaa20d972010-05-17 16:24:16 -0700489static inline void txd_lock(struct dma_async_tx_descriptor *txd)
490{
491}
492static inline void txd_unlock(struct dma_async_tx_descriptor *txd)
493{
494}
495static inline void txd_chain(struct dma_async_tx_descriptor *txd, struct dma_async_tx_descriptor *next)
496{
497 BUG();
498}
499static inline void txd_clear_parent(struct dma_async_tx_descriptor *txd)
500{
501}
502static inline void txd_clear_next(struct dma_async_tx_descriptor *txd)
503{
504}
505static inline struct dma_async_tx_descriptor *txd_next(struct dma_async_tx_descriptor *txd)
506{
507 return NULL;
508}
509static inline struct dma_async_tx_descriptor *txd_parent(struct dma_async_tx_descriptor *txd)
510{
511 return NULL;
512}
513
514#else
515static inline void txd_lock(struct dma_async_tx_descriptor *txd)
516{
517 spin_lock_bh(&txd->lock);
518}
519static inline void txd_unlock(struct dma_async_tx_descriptor *txd)
520{
521 spin_unlock_bh(&txd->lock);
522}
523static inline void txd_chain(struct dma_async_tx_descriptor *txd, struct dma_async_tx_descriptor *next)
524{
525 txd->next = next;
526 next->parent = txd;
527}
528static inline void txd_clear_parent(struct dma_async_tx_descriptor *txd)
529{
530 txd->parent = NULL;
531}
532static inline void txd_clear_next(struct dma_async_tx_descriptor *txd)
533{
534 txd->next = NULL;
535}
536static inline struct dma_async_tx_descriptor *txd_parent(struct dma_async_tx_descriptor *txd)
537{
538 return txd->parent;
539}
540static inline struct dma_async_tx_descriptor *txd_next(struct dma_async_tx_descriptor *txd)
541{
542 return txd->next;
543}
544#endif
545
Chris Leechc13c8262006-05-23 17:18:44 -0700546/**
Linus Walleij07934482010-03-26 16:50:49 -0700547 * struct dma_tx_state - filled in to report the status of
548 * a transfer.
549 * @last: last completed DMA cookie
550 * @used: last issued DMA cookie (i.e. the one in progress)
551 * @residue: the remaining number of bytes left to transmit
552 * on the selected transfer for states DMA_IN_PROGRESS and
553 * DMA_PAUSED if this is implemented in the driver, else 0
554 */
555struct dma_tx_state {
556 dma_cookie_t last;
557 dma_cookie_t used;
558 u32 residue;
559};
560
561/**
Chris Leechc13c8262006-05-23 17:18:44 -0700562 * struct dma_device - info on the entity supplying DMA services
563 * @chancnt: how many DMA channels are supported
Atsushi Nemoto0f571512009-03-06 20:07:14 +0900564 * @privatecnt: how many DMA channels are requested by dma_request_channel
Chris Leechc13c8262006-05-23 17:18:44 -0700565 * @channels: the list of struct dma_chan
566 * @global_node: list_head for global dma_device_list
Dan Williams7405f742007-01-02 11:10:43 -0700567 * @cap_mask: one or more dma_capability flags
568 * @max_xor: maximum number of xor sources, 0 if no capability
Dan Williamsb2f46fd2009-07-14 12:20:36 -0700569 * @max_pq: maximum number of PQ sources and PQ-continue capability
Dan Williams83544ae2009-09-08 17:42:53 -0700570 * @copy_align: alignment shift for memcpy operations
571 * @xor_align: alignment shift for xor operations
572 * @pq_align: alignment shift for pq operations
Randy Dunlapfe4ada22006-07-03 19:44:51 -0700573 * @dev_id: unique device ID
Dan Williams7405f742007-01-02 11:10:43 -0700574 * @dev: struct device reference for dma mapping api
Maxime Ripardcb8cea52014-11-17 14:42:04 +0100575 * @src_addr_widths: bit mask of src addr widths the device supports
576 * @dst_addr_widths: bit mask of dst addr widths the device supports
577 * @directions: bit mask of slave direction the device supports since
578 * the enum dma_transfer_direction is not defined as bits for
579 * each type of direction, the dma controller should fill (1 <<
580 * <TYPE>) and same should be checked by controller as well
581 * @residue_granularity: granularity of the transfer residue reported
582 * by tx_status
Randy Dunlapfe4ada22006-07-03 19:44:51 -0700583 * @device_alloc_chan_resources: allocate resources and return the
584 * number of allocated descriptors
585 * @device_free_chan_resources: release DMA channel's resources
Dan Williams7405f742007-01-02 11:10:43 -0700586 * @device_prep_dma_memcpy: prepares a memcpy operation
587 * @device_prep_dma_xor: prepares a xor operation
Dan Williams099f53c2009-04-08 14:28:37 -0700588 * @device_prep_dma_xor_val: prepares a xor validation operation
Dan Williamsb2f46fd2009-07-14 12:20:36 -0700589 * @device_prep_dma_pq: prepares a pq operation
590 * @device_prep_dma_pq_val: prepares a pqzero_sum operation
Dan Williams7405f742007-01-02 11:10:43 -0700591 * @device_prep_dma_interrupt: prepares an end of chain interrupt operation
Haavard Skinnemoendc0ee6432008-07-08 11:59:35 -0700592 * @device_prep_slave_sg: prepares a slave dma operation
Sascha Hauer782bc952010-09-30 13:56:32 +0000593 * @device_prep_dma_cyclic: prepare a cyclic dma operation suitable for audio.
594 * The function takes a buffer of size buf_len. The callback function will
595 * be called after period_len bytes have been transferred.
Jassi Brarb14dab72011-10-13 12:33:30 +0530596 * @device_prep_interleaved_dma: Transfer expression in a generic way.
Maxime Ripard94a73e32014-11-17 14:42:00 +0100597 * @device_config: Pushes a new configuration to a channel, return 0 or an error
598 * code
Maxime Ripard23a3ea22014-11-17 14:42:01 +0100599 * @device_pause: Pauses any transfer happening on a channel. Returns
600 * 0 or an error code
601 * @device_resume: Resumes any transfer on a channel previously
602 * paused. Returns 0 or an error code
Maxime Ripard7fa0cf42014-11-17 14:42:02 +0100603 * @device_terminate_all: Aborts all transfers on a channel. Returns 0
604 * or an error code
Linus Walleij07934482010-03-26 16:50:49 -0700605 * @device_tx_status: poll for transaction completion, the optional
606 * txstate parameter can be supplied with a pointer to get a
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300607 * struct with auxiliary transfer status information, otherwise the call
Linus Walleij07934482010-03-26 16:50:49 -0700608 * will just return a simple status code
Dan Williams7405f742007-01-02 11:10:43 -0700609 * @device_issue_pending: push pending transactions to hardware
Chris Leechc13c8262006-05-23 17:18:44 -0700610 */
611struct dma_device {
612
613 unsigned int chancnt;
Atsushi Nemoto0f571512009-03-06 20:07:14 +0900614 unsigned int privatecnt;
Chris Leechc13c8262006-05-23 17:18:44 -0700615 struct list_head channels;
616 struct list_head global_node;
Dan Williams7405f742007-01-02 11:10:43 -0700617 dma_cap_mask_t cap_mask;
Dan Williamsb2f46fd2009-07-14 12:20:36 -0700618 unsigned short max_xor;
619 unsigned short max_pq;
Dan Williams83544ae2009-09-08 17:42:53 -0700620 u8 copy_align;
621 u8 xor_align;
622 u8 pq_align;
Dan Williamsb2f46fd2009-07-14 12:20:36 -0700623 #define DMA_HAS_PQ_CONTINUE (1 << 15)
Chris Leechc13c8262006-05-23 17:18:44 -0700624
Chris Leechc13c8262006-05-23 17:18:44 -0700625 int dev_id;
Dan Williams7405f742007-01-02 11:10:43 -0700626 struct device *dev;
Chris Leechc13c8262006-05-23 17:18:44 -0700627
Maxime Ripardcb8cea52014-11-17 14:42:04 +0100628 u32 src_addr_widths;
629 u32 dst_addr_widths;
630 u32 directions;
631 enum dma_residue_granularity residue_granularity;
632
Dan Williamsaa1e6f12009-01-06 11:38:17 -0700633 int (*device_alloc_chan_resources)(struct dma_chan *chan);
Chris Leechc13c8262006-05-23 17:18:44 -0700634 void (*device_free_chan_resources)(struct dma_chan *chan);
Dan Williams7405f742007-01-02 11:10:43 -0700635
636 struct dma_async_tx_descriptor *(*device_prep_dma_memcpy)(
Maxime Ripardceacbdb2014-11-17 14:41:57 +0100637 struct dma_chan *chan, dma_addr_t dst, dma_addr_t src,
Dan Williamsd4c56f92008-02-02 19:49:58 -0700638 size_t len, unsigned long flags);
Dan Williams7405f742007-01-02 11:10:43 -0700639 struct dma_async_tx_descriptor *(*device_prep_dma_xor)(
Maxime Ripardceacbdb2014-11-17 14:41:57 +0100640 struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src,
Dan Williamsd4c56f92008-02-02 19:49:58 -0700641 unsigned int src_cnt, size_t len, unsigned long flags);
Dan Williams099f53c2009-04-08 14:28:37 -0700642 struct dma_async_tx_descriptor *(*device_prep_dma_xor_val)(
Dan Williams00367312008-02-02 19:49:57 -0700643 struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt,
Dan Williamsad283ea2009-08-29 19:09:26 -0700644 size_t len, enum sum_check_flags *result, unsigned long flags);
Dan Williamsb2f46fd2009-07-14 12:20:36 -0700645 struct dma_async_tx_descriptor *(*device_prep_dma_pq)(
646 struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
647 unsigned int src_cnt, const unsigned char *scf,
648 size_t len, unsigned long flags);
649 struct dma_async_tx_descriptor *(*device_prep_dma_pq_val)(
650 struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
651 unsigned int src_cnt, const unsigned char *scf, size_t len,
652 enum sum_check_flags *pqres, unsigned long flags);
Dan Williams7405f742007-01-02 11:10:43 -0700653 struct dma_async_tx_descriptor *(*device_prep_dma_interrupt)(
Dan Williams636bdea2008-04-17 20:17:26 -0700654 struct dma_chan *chan, unsigned long flags);
Ira Snydera86ee032010-09-30 11:46:44 +0000655 struct dma_async_tx_descriptor *(*device_prep_dma_sg)(
656 struct dma_chan *chan,
657 struct scatterlist *dst_sg, unsigned int dst_nents,
658 struct scatterlist *src_sg, unsigned int src_nents,
659 unsigned long flags);
Dan Williams7405f742007-01-02 11:10:43 -0700660
Haavard Skinnemoendc0ee6432008-07-08 11:59:35 -0700661 struct dma_async_tx_descriptor *(*device_prep_slave_sg)(
662 struct dma_chan *chan, struct scatterlist *sgl,
Vinod Koul49920bc2011-10-13 15:15:27 +0530663 unsigned int sg_len, enum dma_transfer_direction direction,
Alexandre Bounine185ecb52012-03-08 15:35:13 -0500664 unsigned long flags, void *context);
Sascha Hauer782bc952010-09-30 13:56:32 +0000665 struct dma_async_tx_descriptor *(*device_prep_dma_cyclic)(
666 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
Alexandre Bounine185ecb52012-03-08 15:35:13 -0500667 size_t period_len, enum dma_transfer_direction direction,
Laurent Pinchart31c1e5a2014-08-01 12:20:10 +0200668 unsigned long flags);
Jassi Brarb14dab72011-10-13 12:33:30 +0530669 struct dma_async_tx_descriptor *(*device_prep_interleaved_dma)(
670 struct dma_chan *chan, struct dma_interleaved_template *xt,
671 unsigned long flags);
Maxime Ripard94a73e32014-11-17 14:42:00 +0100672
673 int (*device_config)(struct dma_chan *chan,
674 struct dma_slave_config *config);
Maxime Ripard23a3ea22014-11-17 14:42:01 +0100675 int (*device_pause)(struct dma_chan *chan);
676 int (*device_resume)(struct dma_chan *chan);
Maxime Ripard7fa0cf42014-11-17 14:42:02 +0100677 int (*device_terminate_all)(struct dma_chan *chan);
Haavard Skinnemoendc0ee6432008-07-08 11:59:35 -0700678
Linus Walleij07934482010-03-26 16:50:49 -0700679 enum dma_status (*device_tx_status)(struct dma_chan *chan,
680 dma_cookie_t cookie,
681 struct dma_tx_state *txstate);
Dan Williams7405f742007-01-02 11:10:43 -0700682 void (*device_issue_pending)(struct dma_chan *chan);
Chris Leechc13c8262006-05-23 17:18:44 -0700683};
684
Sascha Hauer6e3ecaf2010-09-30 13:56:33 +0000685static inline int dmaengine_slave_config(struct dma_chan *chan,
686 struct dma_slave_config *config)
687{
Maxime Ripard94a73e32014-11-17 14:42:00 +0100688 if (chan->device->device_config)
689 return chan->device->device_config(chan, config);
690
Maxime Ripard2c44ad92014-11-17 14:42:54 +0100691 return -ENOSYS;
Sascha Hauer6e3ecaf2010-09-30 13:56:33 +0000692}
693
Andy Shevchenko61cc13a2013-01-10 10:52:56 +0200694static inline bool is_slave_direction(enum dma_transfer_direction direction)
695{
696 return (direction == DMA_MEM_TO_DEV) || (direction == DMA_DEV_TO_MEM);
697}
698
Vinod Koul90b44f82011-07-25 19:57:52 +0530699static inline struct dma_async_tx_descriptor *dmaengine_prep_slave_single(
Kuninori Morimoto922ee082012-04-25 20:50:53 +0200700 struct dma_chan *chan, dma_addr_t buf, size_t len,
Vinod Koul49920bc2011-10-13 15:15:27 +0530701 enum dma_transfer_direction dir, unsigned long flags)
Vinod Koul90b44f82011-07-25 19:57:52 +0530702{
703 struct scatterlist sg;
Kuninori Morimoto922ee082012-04-25 20:50:53 +0200704 sg_init_table(&sg, 1);
705 sg_dma_address(&sg) = buf;
706 sg_dma_len(&sg) = len;
Vinod Koul90b44f82011-07-25 19:57:52 +0530707
Alexandre Bounine185ecb52012-03-08 15:35:13 -0500708 return chan->device->device_prep_slave_sg(chan, &sg, 1,
709 dir, flags, NULL);
Vinod Koul90b44f82011-07-25 19:57:52 +0530710}
711
Alexandre Bounine16052822012-03-08 16:11:18 -0500712static inline struct dma_async_tx_descriptor *dmaengine_prep_slave_sg(
713 struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len,
714 enum dma_transfer_direction dir, unsigned long flags)
715{
716 return chan->device->device_prep_slave_sg(chan, sgl, sg_len,
Alexandre Bounine185ecb52012-03-08 15:35:13 -0500717 dir, flags, NULL);
Alexandre Bounine16052822012-03-08 16:11:18 -0500718}
719
Alexandre Bouninee42d98e2012-05-31 16:26:38 -0700720#ifdef CONFIG_RAPIDIO_DMA_ENGINE
721struct rio_dma_ext;
722static inline struct dma_async_tx_descriptor *dmaengine_prep_rio_sg(
723 struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len,
724 enum dma_transfer_direction dir, unsigned long flags,
725 struct rio_dma_ext *rio_ext)
726{
727 return chan->device->device_prep_slave_sg(chan, sgl, sg_len,
728 dir, flags, rio_ext);
729}
730#endif
731
Alexandre Bounine16052822012-03-08 16:11:18 -0500732static inline struct dma_async_tx_descriptor *dmaengine_prep_dma_cyclic(
733 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
Peter Ujfalusie7736cd2012-09-24 10:58:04 +0300734 size_t period_len, enum dma_transfer_direction dir,
735 unsigned long flags)
Alexandre Bounine16052822012-03-08 16:11:18 -0500736{
737 return chan->device->device_prep_dma_cyclic(chan, buf_addr, buf_len,
Laurent Pinchart31c1e5a2014-08-01 12:20:10 +0200738 period_len, dir, flags);
Sascha Hauer6e3ecaf2010-09-30 13:56:33 +0000739}
740
Barry Songa14acb42012-11-06 21:32:39 +0800741static inline struct dma_async_tx_descriptor *dmaengine_prep_interleaved_dma(
742 struct dma_chan *chan, struct dma_interleaved_template *xt,
743 unsigned long flags)
744{
745 return chan->device->device_prep_interleaved_dma(chan, xt, flags);
746}
747
Vinod Koulb65612a2014-10-11 21:16:43 +0530748static inline struct dma_async_tx_descriptor *dmaengine_prep_dma_sg(
749 struct dma_chan *chan,
750 struct scatterlist *dst_sg, unsigned int dst_nents,
751 struct scatterlist *src_sg, unsigned int src_nents,
752 unsigned long flags)
753{
754 return chan->device->device_prep_dma_sg(chan, dst_sg, dst_nents,
755 src_sg, src_nents, flags);
756}
757
Sascha Hauer6e3ecaf2010-09-30 13:56:33 +0000758static inline int dmaengine_terminate_all(struct dma_chan *chan)
759{
Maxime Ripard7fa0cf42014-11-17 14:42:02 +0100760 if (chan->device->device_terminate_all)
761 return chan->device->device_terminate_all(chan);
762
Maxime Ripard2c44ad92014-11-17 14:42:54 +0100763 return -ENOSYS;
Sascha Hauer6e3ecaf2010-09-30 13:56:33 +0000764}
765
766static inline int dmaengine_pause(struct dma_chan *chan)
767{
Maxime Ripard23a3ea22014-11-17 14:42:01 +0100768 if (chan->device->device_pause)
769 return chan->device->device_pause(chan);
770
Maxime Ripard2c44ad92014-11-17 14:42:54 +0100771 return -ENOSYS;
Sascha Hauer6e3ecaf2010-09-30 13:56:33 +0000772}
773
774static inline int dmaengine_resume(struct dma_chan *chan)
775{
Maxime Ripard23a3ea22014-11-17 14:42:01 +0100776 if (chan->device->device_resume)
777 return chan->device->device_resume(chan);
778
Maxime Ripard2c44ad92014-11-17 14:42:54 +0100779 return -ENOSYS;
Sascha Hauer6e3ecaf2010-09-30 13:56:33 +0000780}
781
Lars-Peter Clausen3052cc22012-06-11 20:11:40 +0200782static inline enum dma_status dmaengine_tx_status(struct dma_chan *chan,
783 dma_cookie_t cookie, struct dma_tx_state *state)
784{
785 return chan->device->device_tx_status(chan, cookie, state);
786}
787
Russell King - ARM Linux98d530f2011-01-01 23:00:23 +0000788static inline dma_cookie_t dmaengine_submit(struct dma_async_tx_descriptor *desc)
Sascha Hauer6e3ecaf2010-09-30 13:56:33 +0000789{
790 return desc->tx_submit(desc);
791}
792
Dan Williams83544ae2009-09-08 17:42:53 -0700793static inline bool dmaengine_check_align(u8 align, size_t off1, size_t off2, size_t len)
794{
795 size_t mask;
796
797 if (!align)
798 return true;
799 mask = (1 << align) - 1;
800 if (mask & (off1 | off2 | len))
801 return false;
802 return true;
803}
804
805static inline bool is_dma_copy_aligned(struct dma_device *dev, size_t off1,
806 size_t off2, size_t len)
807{
808 return dmaengine_check_align(dev->copy_align, off1, off2, len);
809}
810
811static inline bool is_dma_xor_aligned(struct dma_device *dev, size_t off1,
812 size_t off2, size_t len)
813{
814 return dmaengine_check_align(dev->xor_align, off1, off2, len);
815}
816
817static inline bool is_dma_pq_aligned(struct dma_device *dev, size_t off1,
818 size_t off2, size_t len)
819{
820 return dmaengine_check_align(dev->pq_align, off1, off2, len);
821}
822
Dan Williamsb2f46fd2009-07-14 12:20:36 -0700823static inline void
824dma_set_maxpq(struct dma_device *dma, int maxpq, int has_pq_continue)
825{
826 dma->max_pq = maxpq;
827 if (has_pq_continue)
828 dma->max_pq |= DMA_HAS_PQ_CONTINUE;
829}
830
831static inline bool dmaf_continue(enum dma_ctrl_flags flags)
832{
833 return (flags & DMA_PREP_CONTINUE) == DMA_PREP_CONTINUE;
834}
835
836static inline bool dmaf_p_disabled_continue(enum dma_ctrl_flags flags)
837{
838 enum dma_ctrl_flags mask = DMA_PREP_CONTINUE | DMA_PREP_PQ_DISABLE_P;
839
840 return (flags & mask) == mask;
841}
842
843static inline bool dma_dev_has_pq_continue(struct dma_device *dma)
844{
845 return (dma->max_pq & DMA_HAS_PQ_CONTINUE) == DMA_HAS_PQ_CONTINUE;
846}
847
Mathieu Lacaged3f3cf82010-08-14 15:02:44 +0200848static inline unsigned short dma_dev_to_maxpq(struct dma_device *dma)
Dan Williamsb2f46fd2009-07-14 12:20:36 -0700849{
850 return dma->max_pq & ~DMA_HAS_PQ_CONTINUE;
851}
852
853/* dma_maxpq - reduce maxpq in the face of continued operations
854 * @dma - dma device with PQ capability
855 * @flags - to check if DMA_PREP_CONTINUE and DMA_PREP_PQ_DISABLE_P are set
856 *
857 * When an engine does not support native continuation we need 3 extra
858 * source slots to reuse P and Q with the following coefficients:
859 * 1/ {00} * P : remove P from Q', but use it as a source for P'
860 * 2/ {01} * Q : use Q to continue Q' calculation
861 * 3/ {00} * Q : subtract Q from P' to cancel (2)
862 *
863 * In the case where P is disabled we only need 1 extra source:
864 * 1/ {01} * Q : use Q to continue Q' calculation
865 */
866static inline int dma_maxpq(struct dma_device *dma, enum dma_ctrl_flags flags)
867{
868 if (dma_dev_has_pq_continue(dma) || !dmaf_continue(flags))
869 return dma_dev_to_maxpq(dma);
870 else if (dmaf_p_disabled_continue(flags))
871 return dma_dev_to_maxpq(dma) - 1;
872 else if (dmaf_continue(flags))
873 return dma_dev_to_maxpq(dma) - 3;
874 BUG();
875}
876
Chris Leechc13c8262006-05-23 17:18:44 -0700877/* --- public DMA engine API --- */
878
Dan Williams649274d2009-01-11 00:20:39 -0800879#ifdef CONFIG_DMA_ENGINE
Dan Williams209b84a2009-01-06 11:38:17 -0700880void dmaengine_get(void);
881void dmaengine_put(void);
Dan Williams649274d2009-01-11 00:20:39 -0800882#else
883static inline void dmaengine_get(void)
884{
885}
886static inline void dmaengine_put(void)
887{
888}
889#endif
890
Dan Williams729b5d12009-03-25 09:13:25 -0700891#ifdef CONFIG_ASYNC_TX_DMA
892#define async_dmaengine_get() dmaengine_get()
893#define async_dmaengine_put() dmaengine_put()
Dan Williams5fc6d892010-10-07 16:44:50 -0700894#ifndef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH
Dan Williams138f4c32009-09-08 17:42:51 -0700895#define async_dma_find_channel(type) dma_find_channel(DMA_ASYNC_TX)
896#else
Dan Williams729b5d12009-03-25 09:13:25 -0700897#define async_dma_find_channel(type) dma_find_channel(type)
Dan Williams5fc6d892010-10-07 16:44:50 -0700898#endif /* CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH */
Dan Williams729b5d12009-03-25 09:13:25 -0700899#else
900static inline void async_dmaengine_get(void)
901{
902}
903static inline void async_dmaengine_put(void)
904{
905}
906static inline struct dma_chan *
907async_dma_find_channel(enum dma_transaction_type type)
908{
909 return NULL;
910}
Dan Williams138f4c32009-09-08 17:42:51 -0700911#endif /* CONFIG_ASYNC_TX_DMA */
Dan Williams7405f742007-01-02 11:10:43 -0700912void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
Dan Williams7bced392013-12-30 12:37:29 -0800913 struct dma_chan *chan);
Chris Leechc13c8262006-05-23 17:18:44 -0700914
Dan Williams08398752008-07-17 17:59:56 -0700915static inline void async_tx_ack(struct dma_async_tx_descriptor *tx)
Dan Williams7405f742007-01-02 11:10:43 -0700916{
Dan Williams636bdea2008-04-17 20:17:26 -0700917 tx->flags |= DMA_CTRL_ACK;
918}
919
Guennadi Liakhovetskief560682009-01-19 15:36:21 -0700920static inline void async_tx_clear_ack(struct dma_async_tx_descriptor *tx)
921{
922 tx->flags &= ~DMA_CTRL_ACK;
923}
924
Dan Williams08398752008-07-17 17:59:56 -0700925static inline bool async_tx_test_ack(struct dma_async_tx_descriptor *tx)
Dan Williams636bdea2008-04-17 20:17:26 -0700926{
Dan Williams08398752008-07-17 17:59:56 -0700927 return (tx->flags & DMA_CTRL_ACK) == DMA_CTRL_ACK;
Chris Leechc13c8262006-05-23 17:18:44 -0700928}
929
Dan Williams7405f742007-01-02 11:10:43 -0700930#define dma_cap_set(tx, mask) __dma_cap_set((tx), &(mask))
931static inline void
932__dma_cap_set(enum dma_transaction_type tx_type, dma_cap_mask_t *dstp)
933{
934 set_bit(tx_type, dstp->bits);
935}
936
Atsushi Nemoto0f571512009-03-06 20:07:14 +0900937#define dma_cap_clear(tx, mask) __dma_cap_clear((tx), &(mask))
938static inline void
939__dma_cap_clear(enum dma_transaction_type tx_type, dma_cap_mask_t *dstp)
940{
941 clear_bit(tx_type, dstp->bits);
942}
943
Dan Williams33df8ca2009-01-06 11:38:15 -0700944#define dma_cap_zero(mask) __dma_cap_zero(&(mask))
945static inline void __dma_cap_zero(dma_cap_mask_t *dstp)
946{
947 bitmap_zero(dstp->bits, DMA_TX_TYPE_END);
948}
949
Dan Williams7405f742007-01-02 11:10:43 -0700950#define dma_has_cap(tx, mask) __dma_has_cap((tx), &(mask))
951static inline int
952__dma_has_cap(enum dma_transaction_type tx_type, dma_cap_mask_t *srcp)
953{
954 return test_bit(tx_type, srcp->bits);
955}
956
957#define for_each_dma_cap_mask(cap, mask) \
Akinobu Mitae5a087f2012-10-26 23:35:15 +0900958 for_each_set_bit(cap, mask.bits, DMA_TX_TYPE_END)
Dan Williams7405f742007-01-02 11:10:43 -0700959
Chris Leechc13c8262006-05-23 17:18:44 -0700960/**
Dan Williams7405f742007-01-02 11:10:43 -0700961 * dma_async_issue_pending - flush pending transactions to HW
Randy Dunlapfe4ada22006-07-03 19:44:51 -0700962 * @chan: target DMA channel
Chris Leechc13c8262006-05-23 17:18:44 -0700963 *
964 * This allows drivers to push copies to HW in batches,
965 * reducing MMIO writes where possible.
966 */
Dan Williams7405f742007-01-02 11:10:43 -0700967static inline void dma_async_issue_pending(struct dma_chan *chan)
Chris Leechc13c8262006-05-23 17:18:44 -0700968{
Dan Williamsec8670f2008-03-01 07:51:29 -0700969 chan->device->device_issue_pending(chan);
Chris Leechc13c8262006-05-23 17:18:44 -0700970}
971
972/**
Dan Williams7405f742007-01-02 11:10:43 -0700973 * dma_async_is_tx_complete - poll for transaction completion
Chris Leechc13c8262006-05-23 17:18:44 -0700974 * @chan: DMA channel
975 * @cookie: transaction identifier to check status of
976 * @last: returns last completed cookie, can be NULL
977 * @used: returns last issued cookie, can be NULL
978 *
979 * If @last and @used are passed in, upon return they reflect the driver
980 * internal state and can be used with dma_async_is_complete() to check
981 * the status of multiple cookies without re-checking hardware state.
982 */
Dan Williams7405f742007-01-02 11:10:43 -0700983static inline enum dma_status dma_async_is_tx_complete(struct dma_chan *chan,
Chris Leechc13c8262006-05-23 17:18:44 -0700984 dma_cookie_t cookie, dma_cookie_t *last, dma_cookie_t *used)
985{
Linus Walleij07934482010-03-26 16:50:49 -0700986 struct dma_tx_state state;
987 enum dma_status status;
988
989 status = chan->device->device_tx_status(chan, cookie, &state);
990 if (last)
991 *last = state.last;
992 if (used)
993 *used = state.used;
994 return status;
Chris Leechc13c8262006-05-23 17:18:44 -0700995}
996
997/**
998 * dma_async_is_complete - test a cookie against chan state
999 * @cookie: transaction identifier to test status of
1000 * @last_complete: last know completed transaction
1001 * @last_used: last cookie value handed out
1002 *
Bartlomiej Zolnierkiewicze239345f2012-11-08 10:01:01 +00001003 * dma_async_is_complete() is used in dma_async_is_tx_complete()
Sebastian Siewior8a5703f2008-04-21 22:38:45 +00001004 * the test logic is separated for lightweight testing of multiple cookies
Chris Leechc13c8262006-05-23 17:18:44 -07001005 */
1006static inline enum dma_status dma_async_is_complete(dma_cookie_t cookie,
1007 dma_cookie_t last_complete, dma_cookie_t last_used)
1008{
1009 if (last_complete <= last_used) {
1010 if ((cookie <= last_complete) || (cookie > last_used))
Vinod Kouladfedd92013-10-16 13:29:02 +05301011 return DMA_COMPLETE;
Chris Leechc13c8262006-05-23 17:18:44 -07001012 } else {
1013 if ((cookie <= last_complete) && (cookie > last_used))
Vinod Kouladfedd92013-10-16 13:29:02 +05301014 return DMA_COMPLETE;
Chris Leechc13c8262006-05-23 17:18:44 -07001015 }
1016 return DMA_IN_PROGRESS;
1017}
1018
Dan Williamsbca34692010-03-26 16:52:10 -07001019static inline void
1020dma_set_tx_state(struct dma_tx_state *st, dma_cookie_t last, dma_cookie_t used, u32 residue)
1021{
1022 if (st) {
1023 st->last = last;
1024 st->used = used;
1025 st->residue = residue;
1026 }
1027}
1028
Dan Williams07f22112009-01-05 17:14:31 -07001029#ifdef CONFIG_DMA_ENGINE
Jon Mason4a43f392013-09-09 16:51:59 -07001030struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type);
1031enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie);
Dan Williams07f22112009-01-05 17:14:31 -07001032enum dma_status dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx);
Dan Williamsc50331e2009-01-19 15:33:14 -07001033void dma_issue_pending_all(void);
Lars-Peter Clausena53e28d2013-03-25 13:23:52 +01001034struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask,
1035 dma_filter_fn fn, void *fn_param);
Stephen Warren0ad7c002013-11-26 10:04:22 -07001036struct dma_chan *dma_request_slave_channel_reason(struct device *dev,
1037 const char *name);
Markus Pargmannbef29ec2013-02-24 16:36:09 +01001038struct dma_chan *dma_request_slave_channel(struct device *dev, const char *name);
Guennadi Liakhovetski8f33d522010-12-22 14:46:46 +01001039void dma_release_channel(struct dma_chan *chan);
Laurent Pinchartfdb8df92015-01-19 13:54:27 +02001040int dma_get_slave_caps(struct dma_chan *chan, struct dma_slave_caps *caps);
Dan Williams07f22112009-01-05 17:14:31 -07001041#else
Jon Mason4a43f392013-09-09 16:51:59 -07001042static inline struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type)
1043{
1044 return NULL;
1045}
1046static inline enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie)
1047{
Vinod Kouladfedd92013-10-16 13:29:02 +05301048 return DMA_COMPLETE;
Jon Mason4a43f392013-09-09 16:51:59 -07001049}
Dan Williams07f22112009-01-05 17:14:31 -07001050static inline enum dma_status dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx)
1051{
Vinod Kouladfedd92013-10-16 13:29:02 +05301052 return DMA_COMPLETE;
Dan Williams07f22112009-01-05 17:14:31 -07001053}
Dan Williamsc50331e2009-01-19 15:33:14 -07001054static inline void dma_issue_pending_all(void)
1055{
Guennadi Liakhovetski8f33d522010-12-22 14:46:46 +01001056}
Lars-Peter Clausena53e28d2013-03-25 13:23:52 +01001057static inline struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask,
Guennadi Liakhovetski8f33d522010-12-22 14:46:46 +01001058 dma_filter_fn fn, void *fn_param)
1059{
1060 return NULL;
1061}
Stephen Warren0ad7c002013-11-26 10:04:22 -07001062static inline struct dma_chan *dma_request_slave_channel_reason(
1063 struct device *dev, const char *name)
1064{
1065 return ERR_PTR(-ENODEV);
1066}
Jon Hunter9a6cecc2012-09-14 17:41:57 -05001067static inline struct dma_chan *dma_request_slave_channel(struct device *dev,
Markus Pargmannbef29ec2013-02-24 16:36:09 +01001068 const char *name)
Jon Hunter9a6cecc2012-09-14 17:41:57 -05001069{
Vinod Kould18d5f52012-09-25 16:18:55 +05301070 return NULL;
Jon Hunter9a6cecc2012-09-14 17:41:57 -05001071}
Guennadi Liakhovetski8f33d522010-12-22 14:46:46 +01001072static inline void dma_release_channel(struct dma_chan *chan)
1073{
Dan Williamsc50331e2009-01-19 15:33:14 -07001074}
Laurent Pinchartfdb8df92015-01-19 13:54:27 +02001075static inline int dma_get_slave_caps(struct dma_chan *chan,
1076 struct dma_slave_caps *caps)
1077{
1078 return -ENXIO;
1079}
Dan Williams07f22112009-01-05 17:14:31 -07001080#endif
Chris Leechc13c8262006-05-23 17:18:44 -07001081
1082/* --- DMA device --- */
1083
1084int dma_async_device_register(struct dma_device *device);
1085void dma_async_device_unregister(struct dma_device *device);
Dan Williams07f22112009-01-05 17:14:31 -07001086void dma_run_dependencies(struct dma_async_tx_descriptor *tx);
Zhangfei Gao7bb587f2013-06-28 20:39:12 +08001087struct dma_chan *dma_get_slave_channel(struct dma_chan *chan);
Stephen Warren8010dad2013-11-26 12:40:51 -07001088struct dma_chan *dma_get_any_slave_channel(struct dma_device *device);
Dan Williams59b5ec22009-01-06 11:38:15 -07001089#define dma_request_channel(mask, x, y) __dma_request_channel(&(mask), x, y)
Matt Porter864ef692013-02-01 18:22:52 +00001090#define dma_request_slave_channel_compat(mask, x, y, dev, name) \
1091 __dma_request_slave_channel_compat(&(mask), x, y, dev, name)
1092
1093static inline struct dma_chan
Lars-Peter Clausena53e28d2013-03-25 13:23:52 +01001094*__dma_request_slave_channel_compat(const dma_cap_mask_t *mask,
1095 dma_filter_fn fn, void *fn_param,
1096 struct device *dev, char *name)
Matt Porter864ef692013-02-01 18:22:52 +00001097{
1098 struct dma_chan *chan;
1099
1100 chan = dma_request_slave_channel(dev, name);
1101 if (chan)
1102 return chan;
1103
1104 return __dma_request_channel(mask, fn, fn_param);
1105}
Chris Leechc13c8262006-05-23 17:18:44 -07001106#endif /* DMAENGINE_H */