blob: fcfa1939ac41abe768b401f235b8afd7c6d75dbe [file] [log] [blame]
Joerg Roedelf2f45e52009-01-09 12:19:52 +01001/*
2 * Copyright (C) 2008 Advanced Micro Devices, Inc.
3 *
4 * Author: Joerg Roedel <joerg.roedel@amd.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
Joerg Roedel972aa452009-01-09 14:19:54 +010020#include <linux/scatterlist.h>
Joerg Roedel2d62ece2009-01-09 14:10:26 +010021#include <linux/dma-mapping.h>
David Woodhouse6c132d12009-01-19 16:52:39 +010022#include <linux/stacktrace.h>
Joerg Roedelf2f45e52009-01-09 12:19:52 +010023#include <linux/dma-debug.h>
Joerg Roedel30dfa902009-01-09 12:34:49 +010024#include <linux/spinlock.h>
Joerg Roedel788dcfa2009-01-09 13:13:27 +010025#include <linux/debugfs.h>
Joerg Roedel8a6fc702009-05-22 21:23:13 +020026#include <linux/uaccess.h>
Paul Gortmaker23a7bfa2011-07-01 16:23:59 -040027#include <linux/export.h>
Joerg Roedel2d62ece2009-01-09 14:10:26 +010028#include <linux/device.h>
Joerg Roedelf2f45e52009-01-09 12:19:52 +010029#include <linux/types.h>
Joerg Roedel2d62ece2009-01-09 14:10:26 +010030#include <linux/sched.h>
Joerg Roedel8a6fc702009-05-22 21:23:13 +020031#include <linux/ctype.h>
Joerg Roedelf2f45e52009-01-09 12:19:52 +010032#include <linux/list.h>
Joerg Roedel6bf07872009-01-09 12:54:42 +010033#include <linux/slab.h>
Joerg Roedelf2f45e52009-01-09 12:19:52 +010034
Joerg Roedel2e34bde2009-03-16 16:51:55 +010035#include <asm/sections.h>
36
Joerg Roedel30dfa902009-01-09 12:34:49 +010037#define HASH_SIZE 1024ULL
38#define HASH_FN_SHIFT 13
39#define HASH_FN_MASK (HASH_SIZE - 1)
40
Joerg Roedelf2f45e52009-01-09 12:19:52 +010041enum {
42 dma_debug_single,
43 dma_debug_page,
44 dma_debug_sg,
45 dma_debug_coherent,
46};
47
Shuah Khan6c9c6d62012-10-08 11:08:06 -060048enum map_err_types {
49 MAP_ERR_CHECK_NOT_APPLICABLE,
50 MAP_ERR_NOT_CHECKED,
51 MAP_ERR_CHECKED,
52};
53
David Woodhouse6c132d12009-01-19 16:52:39 +010054#define DMA_DEBUG_STACKTRACE_ENTRIES 5
55
Dan Williams0abdd7a2014-01-21 15:48:12 -080056/**
57 * struct dma_debug_entry - track a dma_map* or dma_alloc_coherent mapping
58 * @list: node on pre-allocated free_entries list
59 * @dev: 'dev' argument to dma_map_{page|single|sg} or dma_alloc_coherent
60 * @type: single, page, sg, coherent
61 * @pfn: page frame of the start address
62 * @offset: offset of mapping relative to pfn
63 * @size: length of the mapping
64 * @direction: enum dma_data_direction
65 * @sg_call_ents: 'nents' from dma_map_sg
66 * @sg_mapped_ents: 'mapped_ents' from dma_map_sg
67 * @map_err_type: track whether dma_mapping_error() was checked
68 * @stacktrace: support backtraces when a violation is detected
69 */
Joerg Roedelf2f45e52009-01-09 12:19:52 +010070struct dma_debug_entry {
71 struct list_head list;
72 struct device *dev;
73 int type;
Dan Williams0abdd7a2014-01-21 15:48:12 -080074 unsigned long pfn;
75 size_t offset;
Joerg Roedelf2f45e52009-01-09 12:19:52 +010076 u64 dev_addr;
77 u64 size;
78 int direction;
79 int sg_call_ents;
80 int sg_mapped_ents;
Shuah Khan6c9c6d62012-10-08 11:08:06 -060081 enum map_err_types map_err_type;
David Woodhouse6c132d12009-01-19 16:52:39 +010082#ifdef CONFIG_STACKTRACE
83 struct stack_trace stacktrace;
84 unsigned long st_entries[DMA_DEBUG_STACKTRACE_ENTRIES];
85#endif
Joerg Roedelf2f45e52009-01-09 12:19:52 +010086};
87
Neil Hormanc6a21d02011-08-08 15:13:54 -040088typedef bool (*match_fn)(struct dma_debug_entry *, struct dma_debug_entry *);
89
Joerg Roedel30dfa902009-01-09 12:34:49 +010090struct hash_bucket {
91 struct list_head list;
92 spinlock_t lock;
Joerg Roedel2d62ece2009-01-09 14:10:26 +010093} ____cacheline_aligned_in_smp;
Joerg Roedel30dfa902009-01-09 12:34:49 +010094
95/* Hash list to save the allocated dma addresses */
96static struct hash_bucket dma_entry_hash[HASH_SIZE];
Joerg Roedel3b1e79e2009-01-09 12:42:46 +010097/* List of pre-allocated dma_debug_entry's */
98static LIST_HEAD(free_entries);
99/* Lock for the list above */
100static DEFINE_SPINLOCK(free_entries_lock);
101
102/* Global disable flag - will be set in case of an error */
Viresh Kumar621a5f72015-09-26 15:04:07 -0700103static bool global_disable __read_mostly;
Joerg Roedel3b1e79e2009-01-09 12:42:46 +0100104
Florian Fainelli2ce8e7e2014-12-10 15:41:25 -0800105/* Early initialization disable flag, set at the end of dma_debug_init */
106static bool dma_debug_initialized __read_mostly;
107
Florian Fainelli01ce18b2014-12-10 15:41:23 -0800108static inline bool dma_debug_disabled(void)
109{
Florian Fainelli2ce8e7e2014-12-10 15:41:25 -0800110 return global_disable || !dma_debug_initialized;
Florian Fainelli01ce18b2014-12-10 15:41:23 -0800111}
112
Joerg Roedel788dcfa2009-01-09 13:13:27 +0100113/* Global error count */
114static u32 error_count;
115
116/* Global error show enable*/
117static u32 show_all_errors __read_mostly;
118/* Number of errors to show */
119static u32 show_num_errors = 1;
120
Joerg Roedel3b1e79e2009-01-09 12:42:46 +0100121static u32 num_free_entries;
122static u32 min_free_entries;
FUJITA Tomonorie6a1a892009-04-15 18:22:41 +0900123static u32 nr_total_entries;
Joerg Roedel30dfa902009-01-09 12:34:49 +0100124
Joerg Roedel59d3daa2009-01-09 13:01:56 +0100125/* number of preallocated entries requested by kernel cmdline */
126static u32 req_entries;
127
Joerg Roedel788dcfa2009-01-09 13:13:27 +0100128/* debugfs dentry's for the stuff above */
129static struct dentry *dma_debug_dent __read_mostly;
130static struct dentry *global_disable_dent __read_mostly;
131static struct dentry *error_count_dent __read_mostly;
132static struct dentry *show_all_errors_dent __read_mostly;
133static struct dentry *show_num_errors_dent __read_mostly;
134static struct dentry *num_free_entries_dent __read_mostly;
135static struct dentry *min_free_entries_dent __read_mostly;
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200136static struct dentry *filter_dent __read_mostly;
Joerg Roedel788dcfa2009-01-09 13:13:27 +0100137
Joerg Roedel2e507d82009-05-22 18:24:20 +0200138/* per-driver filter related state */
139
140#define NAME_MAX_LEN 64
141
142static char current_driver_name[NAME_MAX_LEN] __read_mostly;
143static struct device_driver *current_driver __read_mostly;
144
145static DEFINE_RWLOCK(driver_name_lock);
Joerg Roedel30dfa902009-01-09 12:34:49 +0100146
Shuah Khan6c9c6d62012-10-08 11:08:06 -0600147static const char *const maperr2str[] = {
148 [MAP_ERR_CHECK_NOT_APPLICABLE] = "dma map error check not applicable",
149 [MAP_ERR_NOT_CHECKED] = "dma map error not checked",
150 [MAP_ERR_CHECKED] = "dma map error checked",
151};
152
Joerg Roedel2d62ece2009-01-09 14:10:26 +0100153static const char *type2name[4] = { "single", "page",
154 "scather-gather", "coherent" };
155
156static const char *dir2name[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE",
157 "DMA_FROM_DEVICE", "DMA_NONE" };
158
159/*
160 * The access to some variables in this macro is racy. We can't use atomic_t
161 * here because all these variables are exported to debugfs. Some of them even
162 * writeable. This is also the reason why a lock won't help much. But anyway,
163 * the races are no big deal. Here is why:
164 *
165 * error_count: the addition is racy, but the worst thing that can happen is
166 * that we don't count some errors
167 * show_num_errors: the subtraction is racy. Also no big deal because in
168 * worst case this will result in one warning more in the
169 * system log than the user configured. This variable is
170 * writeable via debugfs.
171 */
David Woodhouse6c132d12009-01-19 16:52:39 +0100172static inline void dump_entry_trace(struct dma_debug_entry *entry)
173{
174#ifdef CONFIG_STACKTRACE
175 if (entry) {
Joerg Roedele7ed70e2009-06-08 15:39:24 +0200176 pr_warning("Mapped at:\n");
David Woodhouse6c132d12009-01-19 16:52:39 +0100177 print_stack_trace(&entry->stacktrace, 0);
178 }
179#endif
180}
181
Joerg Roedel2e507d82009-05-22 18:24:20 +0200182static bool driver_filter(struct device *dev)
183{
Joerg Roedel0bf84122009-06-08 15:53:46 +0200184 struct device_driver *drv;
185 unsigned long flags;
186 bool ret;
187
Joerg Roedel2e507d82009-05-22 18:24:20 +0200188 /* driver filter off */
189 if (likely(!current_driver_name[0]))
190 return true;
191
192 /* driver filter on and initialized */
Kyle McMartinec9c96e2009-08-19 21:17:08 -0400193 if (current_driver && dev && dev->driver == current_driver)
Joerg Roedel2e507d82009-05-22 18:24:20 +0200194 return true;
195
Kyle McMartinec9c96e2009-08-19 21:17:08 -0400196 /* driver filter on, but we can't filter on a NULL device... */
197 if (!dev)
198 return false;
199
Joerg Roedel0bf84122009-06-08 15:53:46 +0200200 if (current_driver || !current_driver_name[0])
201 return false;
202
Joerg Roedel2e507d82009-05-22 18:24:20 +0200203 /* driver filter on but not yet initialized */
Alan Sternf3ff9242012-01-24 13:35:24 -0500204 drv = dev->driver;
Joerg Roedel0bf84122009-06-08 15:53:46 +0200205 if (!drv)
206 return false;
Joerg Roedel2e507d82009-05-22 18:24:20 +0200207
Joerg Roedel0bf84122009-06-08 15:53:46 +0200208 /* lock to protect against change of current_driver_name */
209 read_lock_irqsave(&driver_name_lock, flags);
Joerg Roedel2e507d82009-05-22 18:24:20 +0200210
Joerg Roedel0bf84122009-06-08 15:53:46 +0200211 ret = false;
212 if (drv->name &&
213 strncmp(current_driver_name, drv->name, NAME_MAX_LEN - 1) == 0) {
214 current_driver = drv;
215 ret = true;
Joerg Roedel2e507d82009-05-22 18:24:20 +0200216 }
217
Joerg Roedel0bf84122009-06-08 15:53:46 +0200218 read_unlock_irqrestore(&driver_name_lock, flags);
Joerg Roedel0bf84122009-06-08 15:53:46 +0200219
220 return ret;
Joerg Roedel2e507d82009-05-22 18:24:20 +0200221}
222
Kyle McMartinec9c96e2009-08-19 21:17:08 -0400223#define err_printk(dev, entry, format, arg...) do { \
224 error_count += 1; \
225 if (driver_filter(dev) && \
226 (show_all_errors || show_num_errors > 0)) { \
227 WARN(1, "%s %s: " format, \
228 dev ? dev_driver_string(dev) : "NULL", \
229 dev ? dev_name(dev) : "NULL", ## arg); \
230 dump_entry_trace(entry); \
231 } \
232 if (!show_all_errors && show_num_errors > 0) \
233 show_num_errors -= 1; \
Joerg Roedel2d62ece2009-01-09 14:10:26 +0100234 } while (0);
235
Joerg Roedel30dfa902009-01-09 12:34:49 +0100236/*
237 * Hash related functions
238 *
239 * Every DMA-API request is saved into a struct dma_debug_entry. To
240 * have quick access to these structs they are stored into a hash.
241 */
242static int hash_fn(struct dma_debug_entry *entry)
243{
244 /*
245 * Hash function is based on the dma address.
246 * We use bits 20-27 here as the index into the hash
247 */
248 return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK;
249}
250
251/*
252 * Request exclusive access to a hash bucket for a given dma_debug_entry.
253 */
254static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry,
255 unsigned long *flags)
Stephen Boydd5dfc802016-07-26 15:21:08 -0700256 __acquires(&dma_entry_hash[idx].lock)
Joerg Roedel30dfa902009-01-09 12:34:49 +0100257{
258 int idx = hash_fn(entry);
259 unsigned long __flags;
260
261 spin_lock_irqsave(&dma_entry_hash[idx].lock, __flags);
262 *flags = __flags;
263 return &dma_entry_hash[idx];
264}
265
266/*
267 * Give up exclusive access to the hash bucket
268 */
269static void put_hash_bucket(struct hash_bucket *bucket,
270 unsigned long *flags)
Stephen Boydd5dfc802016-07-26 15:21:08 -0700271 __releases(&bucket->lock)
Joerg Roedel30dfa902009-01-09 12:34:49 +0100272{
273 unsigned long __flags = *flags;
274
275 spin_unlock_irqrestore(&bucket->lock, __flags);
276}
277
Neil Hormanc6a21d02011-08-08 15:13:54 -0400278static bool exact_match(struct dma_debug_entry *a, struct dma_debug_entry *b)
279{
Thomas Jarosch91ec37c2011-11-17 20:31:02 +0100280 return ((a->dev_addr == b->dev_addr) &&
Neil Hormanc6a21d02011-08-08 15:13:54 -0400281 (a->dev == b->dev)) ? true : false;
282}
283
284static bool containing_match(struct dma_debug_entry *a,
285 struct dma_debug_entry *b)
286{
287 if (a->dev != b->dev)
288 return false;
289
290 if ((b->dev_addr <= a->dev_addr) &&
291 ((b->dev_addr + b->size) >= (a->dev_addr + a->size)))
292 return true;
293
294 return false;
295}
296
Joerg Roedel30dfa902009-01-09 12:34:49 +0100297/*
298 * Search a given entry in the hash bucket list
299 */
Neil Hormanc6a21d02011-08-08 15:13:54 -0400300static struct dma_debug_entry *__hash_bucket_find(struct hash_bucket *bucket,
301 struct dma_debug_entry *ref,
302 match_fn match)
Joerg Roedel30dfa902009-01-09 12:34:49 +0100303{
Joerg Roedel7caf6a42009-06-05 12:01:35 +0200304 struct dma_debug_entry *entry, *ret = NULL;
Ming Leife73fbe2012-10-19 13:57:01 -0700305 int matches = 0, match_lvl, last_lvl = -1;
Joerg Roedel30dfa902009-01-09 12:34:49 +0100306
307 list_for_each_entry(entry, &bucket->list, list) {
Neil Hormanc6a21d02011-08-08 15:13:54 -0400308 if (!match(ref, entry))
Joerg Roedel7caf6a42009-06-05 12:01:35 +0200309 continue;
310
311 /*
312 * Some drivers map the same physical address multiple
313 * times. Without a hardware IOMMU this results in the
314 * same device addresses being put into the dma-debug
315 * hash multiple times too. This can result in false
André Goddard Rosaaf901ca2009-11-14 13:09:05 -0200316 * positives being reported. Therefore we implement a
Joerg Roedel7caf6a42009-06-05 12:01:35 +0200317 * best-fit algorithm here which returns the entry from
318 * the hash which fits best to the reference value
319 * instead of the first-fit.
320 */
321 matches += 1;
322 match_lvl = 0;
Joerg Roedele5e8c5b2009-06-11 10:03:42 +0200323 entry->size == ref->size ? ++match_lvl : 0;
324 entry->type == ref->type ? ++match_lvl : 0;
325 entry->direction == ref->direction ? ++match_lvl : 0;
326 entry->sg_call_ents == ref->sg_call_ents ? ++match_lvl : 0;
Joerg Roedel7caf6a42009-06-05 12:01:35 +0200327
Joerg Roedele5e8c5b2009-06-11 10:03:42 +0200328 if (match_lvl == 4) {
Joerg Roedel7caf6a42009-06-05 12:01:35 +0200329 /* perfect-fit - return the result */
Joerg Roedel30dfa902009-01-09 12:34:49 +0100330 return entry;
Joerg Roedel7caf6a42009-06-05 12:01:35 +0200331 } else if (match_lvl > last_lvl) {
332 /*
333 * We found an entry that fits better then the
Ming Leife73fbe2012-10-19 13:57:01 -0700334 * previous one or it is the 1st match.
Joerg Roedel7caf6a42009-06-05 12:01:35 +0200335 */
336 last_lvl = match_lvl;
337 ret = entry;
338 }
Joerg Roedel30dfa902009-01-09 12:34:49 +0100339 }
340
Joerg Roedel7caf6a42009-06-05 12:01:35 +0200341 /*
342 * If we have multiple matches but no perfect-fit, just return
343 * NULL.
344 */
345 ret = (matches == 1) ? ret : NULL;
346
347 return ret;
Joerg Roedel30dfa902009-01-09 12:34:49 +0100348}
349
Neil Hormanc6a21d02011-08-08 15:13:54 -0400350static struct dma_debug_entry *bucket_find_exact(struct hash_bucket *bucket,
351 struct dma_debug_entry *ref)
352{
353 return __hash_bucket_find(bucket, ref, exact_match);
354}
355
356static struct dma_debug_entry *bucket_find_contain(struct hash_bucket **bucket,
357 struct dma_debug_entry *ref,
358 unsigned long *flags)
359{
360
361 unsigned int max_range = dma_get_max_seg_size(ref->dev);
362 struct dma_debug_entry *entry, index = *ref;
363 unsigned int range = 0;
364
365 while (range <= max_range) {
Sebastian Otta7a2c02a42015-04-16 12:43:25 -0700366 entry = __hash_bucket_find(*bucket, ref, containing_match);
Neil Hormanc6a21d02011-08-08 15:13:54 -0400367
368 if (entry)
369 return entry;
370
371 /*
372 * Nothing found, go back a hash bucket
373 */
374 put_hash_bucket(*bucket, flags);
375 range += (1 << HASH_FN_SHIFT);
376 index.dev_addr -= (1 << HASH_FN_SHIFT);
377 *bucket = get_hash_bucket(&index, flags);
378 }
379
380 return NULL;
381}
382
Joerg Roedel30dfa902009-01-09 12:34:49 +0100383/*
384 * Add an entry to a hash bucket
385 */
386static void hash_bucket_add(struct hash_bucket *bucket,
387 struct dma_debug_entry *entry)
388{
389 list_add_tail(&entry->list, &bucket->list);
390}
391
392/*
393 * Remove entry from a hash bucket list
394 */
395static void hash_bucket_del(struct dma_debug_entry *entry)
396{
397 list_del(&entry->list);
398}
399
Dan Williams0abdd7a2014-01-21 15:48:12 -0800400static unsigned long long phys_addr(struct dma_debug_entry *entry)
401{
402 return page_to_phys(pfn_to_page(entry->pfn)) + entry->offset;
403}
404
Joerg Roedel30dfa902009-01-09 12:34:49 +0100405/*
David Woodhouseac26c182009-02-12 16:19:13 +0100406 * Dump mapping entries for debugging purposes
407 */
408void debug_dma_dump_mappings(struct device *dev)
409{
410 int idx;
411
412 for (idx = 0; idx < HASH_SIZE; idx++) {
413 struct hash_bucket *bucket = &dma_entry_hash[idx];
414 struct dma_debug_entry *entry;
415 unsigned long flags;
416
417 spin_lock_irqsave(&bucket->lock, flags);
418
419 list_for_each_entry(entry, &bucket->list, list) {
420 if (!dev || dev == entry->dev) {
421 dev_info(entry->dev,
Dan Williams0abdd7a2014-01-21 15:48:12 -0800422 "%s idx %d P=%Lx N=%lx D=%Lx L=%Lx %s %s\n",
David Woodhouseac26c182009-02-12 16:19:13 +0100423 type2name[entry->type], idx,
Dan Williams0abdd7a2014-01-21 15:48:12 -0800424 phys_addr(entry), entry->pfn,
David Woodhouseac26c182009-02-12 16:19:13 +0100425 entry->dev_addr, entry->size,
Shuah Khan6c9c6d62012-10-08 11:08:06 -0600426 dir2name[entry->direction],
427 maperr2str[entry->map_err_type]);
David Woodhouseac26c182009-02-12 16:19:13 +0100428 }
429 }
430
431 spin_unlock_irqrestore(&bucket->lock, flags);
432 }
433}
434EXPORT_SYMBOL(debug_dma_dump_mappings);
435
436/*
Dan Williams3b7a6412014-03-03 15:38:21 -0800437 * For each mapping (initial cacheline in the case of
438 * dma_alloc_coherent/dma_map_page, initial cacheline in each page of a
439 * scatterlist, or the cacheline specified in dma_map_single) insert
440 * into this tree using the cacheline as the key. At
Dan Williams0abdd7a2014-01-21 15:48:12 -0800441 * dma_unmap_{single|sg|page} or dma_free_coherent delete the entry. If
Dan Williams3b7a6412014-03-03 15:38:21 -0800442 * the entry already exists at insertion time add a tag as a reference
Dan Williams0abdd7a2014-01-21 15:48:12 -0800443 * count for the overlapping mappings. For now, the overlap tracking
Dan Williams3b7a6412014-03-03 15:38:21 -0800444 * just ensures that 'unmaps' balance 'maps' before marking the
445 * cacheline idle, but we should also be flagging overlaps as an API
446 * violation.
Dan Williams0abdd7a2014-01-21 15:48:12 -0800447 *
448 * Memory usage is mostly constrained by the maximum number of available
449 * dma-debug entries in that we need a free dma_debug_entry before
Dan Williams3b7a6412014-03-03 15:38:21 -0800450 * inserting into the tree. In the case of dma_map_page and
451 * dma_alloc_coherent there is only one dma_debug_entry and one
452 * dma_active_cacheline entry to track per event. dma_map_sg(), on the
453 * other hand, consumes a single dma_debug_entry, but inserts 'nents'
454 * entries into the tree.
Dan Williams0abdd7a2014-01-21 15:48:12 -0800455 *
456 * At any time debug_dma_assert_idle() can be called to trigger a
Dan Williams3b7a6412014-03-03 15:38:21 -0800457 * warning if any cachelines in the given page are in the active set.
Dan Williams0abdd7a2014-01-21 15:48:12 -0800458 */
Dan Williams3b7a6412014-03-03 15:38:21 -0800459static RADIX_TREE(dma_active_cacheline, GFP_NOWAIT);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800460static DEFINE_SPINLOCK(radix_lock);
Dan Williams3b7a6412014-03-03 15:38:21 -0800461#define ACTIVE_CACHELINE_MAX_OVERLAP ((1 << RADIX_TREE_MAX_TAGS) - 1)
462#define CACHELINE_PER_PAGE_SHIFT (PAGE_SHIFT - L1_CACHE_SHIFT)
463#define CACHELINES_PER_PAGE (1 << CACHELINE_PER_PAGE_SHIFT)
Dan Williams0abdd7a2014-01-21 15:48:12 -0800464
Dan Williams3b7a6412014-03-03 15:38:21 -0800465static phys_addr_t to_cacheline_number(struct dma_debug_entry *entry)
466{
467 return (entry->pfn << CACHELINE_PER_PAGE_SHIFT) +
468 (entry->offset >> L1_CACHE_SHIFT);
469}
470
471static int active_cacheline_read_overlap(phys_addr_t cln)
Dan Williams0abdd7a2014-01-21 15:48:12 -0800472{
473 int overlap = 0, i;
474
475 for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--)
Dan Williams3b7a6412014-03-03 15:38:21 -0800476 if (radix_tree_tag_get(&dma_active_cacheline, cln, i))
Dan Williams0abdd7a2014-01-21 15:48:12 -0800477 overlap |= 1 << i;
478 return overlap;
479}
480
Dan Williams3b7a6412014-03-03 15:38:21 -0800481static int active_cacheline_set_overlap(phys_addr_t cln, int overlap)
Dan Williams0abdd7a2014-01-21 15:48:12 -0800482{
483 int i;
484
Dan Williams3b7a6412014-03-03 15:38:21 -0800485 if (overlap > ACTIVE_CACHELINE_MAX_OVERLAP || overlap < 0)
Dan Williams59f2e7d2014-01-29 14:05:53 -0800486 return overlap;
Dan Williams0abdd7a2014-01-21 15:48:12 -0800487
488 for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--)
489 if (overlap & 1 << i)
Dan Williams3b7a6412014-03-03 15:38:21 -0800490 radix_tree_tag_set(&dma_active_cacheline, cln, i);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800491 else
Dan Williams3b7a6412014-03-03 15:38:21 -0800492 radix_tree_tag_clear(&dma_active_cacheline, cln, i);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800493
494 return overlap;
495}
496
Dan Williams3b7a6412014-03-03 15:38:21 -0800497static void active_cacheline_inc_overlap(phys_addr_t cln)
Dan Williams0abdd7a2014-01-21 15:48:12 -0800498{
Dan Williams3b7a6412014-03-03 15:38:21 -0800499 int overlap = active_cacheline_read_overlap(cln);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800500
Dan Williams3b7a6412014-03-03 15:38:21 -0800501 overlap = active_cacheline_set_overlap(cln, ++overlap);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800502
503 /* If we overflowed the overlap counter then we're potentially
504 * leaking dma-mappings. Otherwise, if maps and unmaps are
505 * balanced then this overflow may cause false negatives in
Dan Williams3b7a6412014-03-03 15:38:21 -0800506 * debug_dma_assert_idle() as the cacheline may be marked idle
Dan Williams0abdd7a2014-01-21 15:48:12 -0800507 * prematurely.
508 */
Dan Williams3b7a6412014-03-03 15:38:21 -0800509 WARN_ONCE(overlap > ACTIVE_CACHELINE_MAX_OVERLAP,
510 "DMA-API: exceeded %d overlapping mappings of cacheline %pa\n",
511 ACTIVE_CACHELINE_MAX_OVERLAP, &cln);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800512}
513
Dan Williams3b7a6412014-03-03 15:38:21 -0800514static int active_cacheline_dec_overlap(phys_addr_t cln)
Dan Williams0abdd7a2014-01-21 15:48:12 -0800515{
Dan Williams3b7a6412014-03-03 15:38:21 -0800516 int overlap = active_cacheline_read_overlap(cln);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800517
Dan Williams3b7a6412014-03-03 15:38:21 -0800518 return active_cacheline_set_overlap(cln, --overlap);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800519}
520
Dan Williams3b7a6412014-03-03 15:38:21 -0800521static int active_cacheline_insert(struct dma_debug_entry *entry)
Dan Williams0abdd7a2014-01-21 15:48:12 -0800522{
Dan Williams3b7a6412014-03-03 15:38:21 -0800523 phys_addr_t cln = to_cacheline_number(entry);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800524 unsigned long flags;
525 int rc;
526
Dan Williams3b7a6412014-03-03 15:38:21 -0800527 /* If the device is not writing memory then we don't have any
528 * concerns about the cpu consuming stale data. This mitigates
529 * legitimate usages of overlapping mappings.
530 */
531 if (entry->direction == DMA_TO_DEVICE)
532 return 0;
533
Dan Williams0abdd7a2014-01-21 15:48:12 -0800534 spin_lock_irqsave(&radix_lock, flags);
Dan Williams3b7a6412014-03-03 15:38:21 -0800535 rc = radix_tree_insert(&dma_active_cacheline, cln, entry);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800536 if (rc == -EEXIST)
Dan Williams3b7a6412014-03-03 15:38:21 -0800537 active_cacheline_inc_overlap(cln);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800538 spin_unlock_irqrestore(&radix_lock, flags);
539
540 return rc;
541}
542
Dan Williams3b7a6412014-03-03 15:38:21 -0800543static void active_cacheline_remove(struct dma_debug_entry *entry)
Dan Williams0abdd7a2014-01-21 15:48:12 -0800544{
Dan Williams3b7a6412014-03-03 15:38:21 -0800545 phys_addr_t cln = to_cacheline_number(entry);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800546 unsigned long flags;
547
Dan Williams3b7a6412014-03-03 15:38:21 -0800548 /* ...mirror the insert case */
549 if (entry->direction == DMA_TO_DEVICE)
550 return;
551
Dan Williams0abdd7a2014-01-21 15:48:12 -0800552 spin_lock_irqsave(&radix_lock, flags);
Dan Williams59f2e7d2014-01-29 14:05:53 -0800553 /* since we are counting overlaps the final put of the
Dan Williams3b7a6412014-03-03 15:38:21 -0800554 * cacheline will occur when the overlap count is 0.
555 * active_cacheline_dec_overlap() returns -1 in that case
Dan Williams59f2e7d2014-01-29 14:05:53 -0800556 */
Dan Williams3b7a6412014-03-03 15:38:21 -0800557 if (active_cacheline_dec_overlap(cln) < 0)
558 radix_tree_delete(&dma_active_cacheline, cln);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800559 spin_unlock_irqrestore(&radix_lock, flags);
560}
561
562/**
563 * debug_dma_assert_idle() - assert that a page is not undergoing dma
Dan Williams3b7a6412014-03-03 15:38:21 -0800564 * @page: page to lookup in the dma_active_cacheline tree
Dan Williams0abdd7a2014-01-21 15:48:12 -0800565 *
566 * Place a call to this routine in cases where the cpu touching the page
567 * before the dma completes (page is dma_unmapped) will lead to data
568 * corruption.
569 */
570void debug_dma_assert_idle(struct page *page)
571{
Dan Williams3b7a6412014-03-03 15:38:21 -0800572 static struct dma_debug_entry *ents[CACHELINES_PER_PAGE];
573 struct dma_debug_entry *entry = NULL;
574 void **results = (void **) &ents;
575 unsigned int nents, i;
Dan Williams0abdd7a2014-01-21 15:48:12 -0800576 unsigned long flags;
Dan Williams3b7a6412014-03-03 15:38:21 -0800577 phys_addr_t cln;
Dan Williams0abdd7a2014-01-21 15:48:12 -0800578
Haggai Eranc9d120b2015-07-17 16:24:06 -0700579 if (dma_debug_disabled())
580 return;
581
Dan Williams0abdd7a2014-01-21 15:48:12 -0800582 if (!page)
583 return;
584
Dan Williams3b7a6412014-03-03 15:38:21 -0800585 cln = (phys_addr_t) page_to_pfn(page) << CACHELINE_PER_PAGE_SHIFT;
Dan Williams0abdd7a2014-01-21 15:48:12 -0800586 spin_lock_irqsave(&radix_lock, flags);
Dan Williams3b7a6412014-03-03 15:38:21 -0800587 nents = radix_tree_gang_lookup(&dma_active_cacheline, results, cln,
588 CACHELINES_PER_PAGE);
589 for (i = 0; i < nents; i++) {
590 phys_addr_t ent_cln = to_cacheline_number(ents[i]);
591
592 if (ent_cln == cln) {
593 entry = ents[i];
594 break;
595 } else if (ent_cln >= cln + CACHELINES_PER_PAGE)
596 break;
597 }
Dan Williams0abdd7a2014-01-21 15:48:12 -0800598 spin_unlock_irqrestore(&radix_lock, flags);
599
600 if (!entry)
601 return;
602
Dan Williams3b7a6412014-03-03 15:38:21 -0800603 cln = to_cacheline_number(entry);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800604 err_printk(entry->dev, entry,
Dan Williams3b7a6412014-03-03 15:38:21 -0800605 "DMA-API: cpu touching an active dma mapped cacheline [cln=%pa]\n",
606 &cln);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800607}
608
609/*
Joerg Roedel30dfa902009-01-09 12:34:49 +0100610 * Wrapper function for adding an entry to the hash.
611 * This function takes care of locking itself.
612 */
613static void add_dma_entry(struct dma_debug_entry *entry)
614{
615 struct hash_bucket *bucket;
616 unsigned long flags;
Dan Williams0abdd7a2014-01-21 15:48:12 -0800617 int rc;
Joerg Roedel30dfa902009-01-09 12:34:49 +0100618
619 bucket = get_hash_bucket(entry, &flags);
620 hash_bucket_add(bucket, entry);
621 put_hash_bucket(bucket, &flags);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800622
Dan Williams3b7a6412014-03-03 15:38:21 -0800623 rc = active_cacheline_insert(entry);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800624 if (rc == -ENOMEM) {
Dan Williams3b7a6412014-03-03 15:38:21 -0800625 pr_err("DMA-API: cacheline tracking ENOMEM, dma-debug disabled\n");
Dan Williams0abdd7a2014-01-21 15:48:12 -0800626 global_disable = true;
627 }
628
629 /* TODO: report -EEXIST errors here as overlapping mappings are
630 * not supported by the DMA API
631 */
Joerg Roedel30dfa902009-01-09 12:34:49 +0100632}
633
FUJITA Tomonorie6a1a892009-04-15 18:22:41 +0900634static struct dma_debug_entry *__dma_entry_alloc(void)
635{
636 struct dma_debug_entry *entry;
637
638 entry = list_entry(free_entries.next, struct dma_debug_entry, list);
639 list_del(&entry->list);
640 memset(entry, 0, sizeof(*entry));
641
642 num_free_entries -= 1;
643 if (num_free_entries < min_free_entries)
644 min_free_entries = num_free_entries;
645
646 return entry;
647}
648
Joerg Roedel3b1e79e2009-01-09 12:42:46 +0100649/* struct dma_entry allocator
650 *
651 * The next two functions implement the allocator for
652 * struct dma_debug_entries.
653 */
654static struct dma_debug_entry *dma_entry_alloc(void)
655{
Jakub Kicinski29cdd4e2012-04-04 03:19:10 +0200656 struct dma_debug_entry *entry;
Joerg Roedel3b1e79e2009-01-09 12:42:46 +0100657 unsigned long flags;
658
659 spin_lock_irqsave(&free_entries_lock, flags);
660
661 if (list_empty(&free_entries)) {
Joerg Roedel3b1e79e2009-01-09 12:42:46 +0100662 global_disable = true;
Jakub Kicinski29cdd4e2012-04-04 03:19:10 +0200663 spin_unlock_irqrestore(&free_entries_lock, flags);
Ville Syrjälä3017cd62016-05-26 15:16:25 -0700664 pr_err("DMA-API: debugging out of memory - disabling\n");
Jakub Kicinski29cdd4e2012-04-04 03:19:10 +0200665 return NULL;
Joerg Roedel3b1e79e2009-01-09 12:42:46 +0100666 }
667
FUJITA Tomonorie6a1a892009-04-15 18:22:41 +0900668 entry = __dma_entry_alloc();
Joerg Roedel3b1e79e2009-01-09 12:42:46 +0100669
Jakub Kicinski29cdd4e2012-04-04 03:19:10 +0200670 spin_unlock_irqrestore(&free_entries_lock, flags);
671
David Woodhouse6c132d12009-01-19 16:52:39 +0100672#ifdef CONFIG_STACKTRACE
673 entry->stacktrace.max_entries = DMA_DEBUG_STACKTRACE_ENTRIES;
674 entry->stacktrace.entries = entry->st_entries;
675 entry->stacktrace.skip = 2;
676 save_stack_trace(&entry->stacktrace);
677#endif
Joerg Roedel3b1e79e2009-01-09 12:42:46 +0100678
Joerg Roedel3b1e79e2009-01-09 12:42:46 +0100679 return entry;
680}
681
682static void dma_entry_free(struct dma_debug_entry *entry)
683{
684 unsigned long flags;
685
Dan Williams3b7a6412014-03-03 15:38:21 -0800686 active_cacheline_remove(entry);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800687
Joerg Roedel3b1e79e2009-01-09 12:42:46 +0100688 /*
689 * add to beginning of the list - this way the entries are
690 * more likely cache hot when they are reallocated.
691 */
692 spin_lock_irqsave(&free_entries_lock, flags);
693 list_add(&entry->list, &free_entries);
694 num_free_entries += 1;
695 spin_unlock_irqrestore(&free_entries_lock, flags);
696}
697
FUJITA Tomonorie6a1a892009-04-15 18:22:41 +0900698int dma_debug_resize_entries(u32 num_entries)
699{
700 int i, delta, ret = 0;
701 unsigned long flags;
702 struct dma_debug_entry *entry;
703 LIST_HEAD(tmp);
704
705 spin_lock_irqsave(&free_entries_lock, flags);
706
707 if (nr_total_entries < num_entries) {
708 delta = num_entries - nr_total_entries;
709
710 spin_unlock_irqrestore(&free_entries_lock, flags);
711
712 for (i = 0; i < delta; i++) {
713 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
714 if (!entry)
715 break;
716
717 list_add_tail(&entry->list, &tmp);
718 }
719
720 spin_lock_irqsave(&free_entries_lock, flags);
721
722 list_splice(&tmp, &free_entries);
723 nr_total_entries += i;
724 num_free_entries += i;
725 } else {
726 delta = nr_total_entries - num_entries;
727
728 for (i = 0; i < delta && !list_empty(&free_entries); i++) {
729 entry = __dma_entry_alloc();
730 kfree(entry);
731 }
732
733 nr_total_entries -= i;
734 }
735
736 if (nr_total_entries != num_entries)
737 ret = 1;
738
739 spin_unlock_irqrestore(&free_entries_lock, flags);
740
741 return ret;
742}
743EXPORT_SYMBOL(dma_debug_resize_entries);
744
Joerg Roedel6bf07872009-01-09 12:54:42 +0100745/*
746 * DMA-API debugging init code
747 *
748 * The init code does two things:
749 * 1. Initialize core data structures
750 * 2. Preallocate a given number of dma_debug_entry structs
751 */
752
753static int prealloc_memory(u32 num_entries)
754{
755 struct dma_debug_entry *entry, *next_entry;
756 int i;
757
758 for (i = 0; i < num_entries; ++i) {
759 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
760 if (!entry)
761 goto out_err;
762
763 list_add_tail(&entry->list, &free_entries);
764 }
765
766 num_free_entries = num_entries;
767 min_free_entries = num_entries;
768
Joerg Roedele7ed70e2009-06-08 15:39:24 +0200769 pr_info("DMA-API: preallocated %d debug entries\n", num_entries);
Joerg Roedel6bf07872009-01-09 12:54:42 +0100770
771 return 0;
772
773out_err:
774
775 list_for_each_entry_safe(entry, next_entry, &free_entries, list) {
776 list_del(&entry->list);
777 kfree(entry);
778 }
779
780 return -ENOMEM;
781}
782
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200783static ssize_t filter_read(struct file *file, char __user *user_buf,
784 size_t count, loff_t *ppos)
785{
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200786 char buf[NAME_MAX_LEN + 1];
Joerg Roedelc17e2cf2009-06-08 15:19:29 +0200787 unsigned long flags;
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200788 int len;
789
790 if (!current_driver_name[0])
791 return 0;
792
793 /*
794 * We can't copy to userspace directly because current_driver_name can
795 * only be read under the driver_name_lock with irqs disabled. So
796 * create a temporary copy first.
797 */
798 read_lock_irqsave(&driver_name_lock, flags);
799 len = scnprintf(buf, NAME_MAX_LEN + 1, "%s\n", current_driver_name);
800 read_unlock_irqrestore(&driver_name_lock, flags);
801
802 return simple_read_from_buffer(user_buf, count, ppos, buf, len);
803}
804
805static ssize_t filter_write(struct file *file, const char __user *userbuf,
806 size_t count, loff_t *ppos)
807{
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200808 char buf[NAME_MAX_LEN];
Joerg Roedelc17e2cf2009-06-08 15:19:29 +0200809 unsigned long flags;
810 size_t len;
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200811 int i;
812
813 /*
814 * We can't copy from userspace directly. Access to
815 * current_driver_name is protected with a write_lock with irqs
816 * disabled. Since copy_from_user can fault and may sleep we
817 * need to copy to temporary buffer first
818 */
Joerg Roedele7ed70e2009-06-08 15:39:24 +0200819 len = min(count, (size_t)(NAME_MAX_LEN - 1));
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200820 if (copy_from_user(buf, userbuf, len))
821 return -EFAULT;
822
823 buf[len] = 0;
824
825 write_lock_irqsave(&driver_name_lock, flags);
826
Joerg Roedel312325092009-06-08 15:07:08 +0200827 /*
828 * Now handle the string we got from userspace very carefully.
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200829 * The rules are:
830 * - only use the first token we got
831 * - token delimiter is everything looking like a space
832 * character (' ', '\n', '\t' ...)
833 *
834 */
835 if (!isalnum(buf[0])) {
836 /*
Joerg Roedel312325092009-06-08 15:07:08 +0200837 * If the first character userspace gave us is not
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200838 * alphanumerical then assume the filter should be
839 * switched off.
840 */
841 if (current_driver_name[0])
Joerg Roedele7ed70e2009-06-08 15:39:24 +0200842 pr_info("DMA-API: switching off dma-debug driver filter\n");
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200843 current_driver_name[0] = 0;
844 current_driver = NULL;
845 goto out_unlock;
846 }
847
848 /*
849 * Now parse out the first token and use it as the name for the
850 * driver to filter for.
851 */
Dan Carpenter39a37ce2010-04-06 19:45:12 +0300852 for (i = 0; i < NAME_MAX_LEN - 1; ++i) {
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200853 current_driver_name[i] = buf[i];
854 if (isspace(buf[i]) || buf[i] == ' ' || buf[i] == 0)
855 break;
856 }
857 current_driver_name[i] = 0;
858 current_driver = NULL;
859
Joerg Roedele7ed70e2009-06-08 15:39:24 +0200860 pr_info("DMA-API: enable driver filter for driver [%s]\n",
861 current_driver_name);
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200862
863out_unlock:
864 write_unlock_irqrestore(&driver_name_lock, flags);
865
866 return count;
867}
868
Thiago Farinaaeb583d2010-01-18 18:57:33 -0500869static const struct file_operations filter_fops = {
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200870 .read = filter_read,
871 .write = filter_write,
Arnd Bergmann6038f372010-08-15 18:52:59 +0200872 .llseek = default_llseek,
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200873};
874
Joerg Roedel788dcfa2009-01-09 13:13:27 +0100875static int dma_debug_fs_init(void)
876{
877 dma_debug_dent = debugfs_create_dir("dma-api", NULL);
878 if (!dma_debug_dent) {
Joerg Roedele7ed70e2009-06-08 15:39:24 +0200879 pr_err("DMA-API: can not create debugfs directory\n");
Joerg Roedel788dcfa2009-01-09 13:13:27 +0100880 return -ENOMEM;
881 }
882
883 global_disable_dent = debugfs_create_bool("disabled", 0444,
884 dma_debug_dent,
Dan Carpenter68ee6d22012-06-27 12:08:55 +0300885 &global_disable);
Joerg Roedel788dcfa2009-01-09 13:13:27 +0100886 if (!global_disable_dent)
887 goto out_err;
888
889 error_count_dent = debugfs_create_u32("error_count", 0444,
890 dma_debug_dent, &error_count);
891 if (!error_count_dent)
892 goto out_err;
893
894 show_all_errors_dent = debugfs_create_u32("all_errors", 0644,
895 dma_debug_dent,
896 &show_all_errors);
897 if (!show_all_errors_dent)
898 goto out_err;
899
900 show_num_errors_dent = debugfs_create_u32("num_errors", 0644,
901 dma_debug_dent,
902 &show_num_errors);
903 if (!show_num_errors_dent)
904 goto out_err;
905
906 num_free_entries_dent = debugfs_create_u32("num_free_entries", 0444,
907 dma_debug_dent,
908 &num_free_entries);
909 if (!num_free_entries_dent)
910 goto out_err;
911
912 min_free_entries_dent = debugfs_create_u32("min_free_entries", 0444,
913 dma_debug_dent,
914 &min_free_entries);
915 if (!min_free_entries_dent)
916 goto out_err;
917
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200918 filter_dent = debugfs_create_file("driver_filter", 0644,
919 dma_debug_dent, NULL, &filter_fops);
920 if (!filter_dent)
921 goto out_err;
922
Joerg Roedel788dcfa2009-01-09 13:13:27 +0100923 return 0;
924
925out_err:
926 debugfs_remove_recursive(dma_debug_dent);
927
928 return -ENOMEM;
929}
930
Stanislaw Gruszkaba4b87ad2011-03-31 08:08:09 -0400931static int device_dma_allocations(struct device *dev, struct dma_debug_entry **out_entry)
Joerg Roedeled888ae2009-05-22 17:16:04 +0200932{
933 struct dma_debug_entry *entry;
934 unsigned long flags;
935 int count = 0, i;
936
Joerg Roedelbe81c6e2009-06-08 15:46:19 +0200937 local_irq_save(flags);
938
Joerg Roedeled888ae2009-05-22 17:16:04 +0200939 for (i = 0; i < HASH_SIZE; ++i) {
Joerg Roedelbe81c6e2009-06-08 15:46:19 +0200940 spin_lock(&dma_entry_hash[i].lock);
Joerg Roedeled888ae2009-05-22 17:16:04 +0200941 list_for_each_entry(entry, &dma_entry_hash[i].list, list) {
Stanislaw Gruszkaba4b87ad2011-03-31 08:08:09 -0400942 if (entry->dev == dev) {
Joerg Roedeled888ae2009-05-22 17:16:04 +0200943 count += 1;
Stanislaw Gruszkaba4b87ad2011-03-31 08:08:09 -0400944 *out_entry = entry;
945 }
Joerg Roedeled888ae2009-05-22 17:16:04 +0200946 }
Joerg Roedelbe81c6e2009-06-08 15:46:19 +0200947 spin_unlock(&dma_entry_hash[i].lock);
Joerg Roedeled888ae2009-05-22 17:16:04 +0200948 }
949
Joerg Roedelbe81c6e2009-06-08 15:46:19 +0200950 local_irq_restore(flags);
951
Joerg Roedeled888ae2009-05-22 17:16:04 +0200952 return count;
953}
954
Ingo Molnara8fe9ea2009-12-31 15:16:23 +0100955static int dma_debug_device_change(struct notifier_block *nb, unsigned long action, void *data)
Joerg Roedeled888ae2009-05-22 17:16:04 +0200956{
957 struct device *dev = data;
Stanislaw Gruszkaba4b87ad2011-03-31 08:08:09 -0400958 struct dma_debug_entry *uninitialized_var(entry);
Joerg Roedeled888ae2009-05-22 17:16:04 +0200959 int count;
960
Florian Fainelli01ce18b2014-12-10 15:41:23 -0800961 if (dma_debug_disabled())
Ingo Molnara8fe9ea2009-12-31 15:16:23 +0100962 return 0;
Joerg Roedeled888ae2009-05-22 17:16:04 +0200963
964 switch (action) {
965 case BUS_NOTIFY_UNBOUND_DRIVER:
Stanislaw Gruszkaba4b87ad2011-03-31 08:08:09 -0400966 count = device_dma_allocations(dev, &entry);
Joerg Roedeled888ae2009-05-22 17:16:04 +0200967 if (count == 0)
968 break;
Stanislaw Gruszkaba4b87ad2011-03-31 08:08:09 -0400969 err_printk(dev, entry, "DMA-API: device driver has pending "
Joerg Roedeled888ae2009-05-22 17:16:04 +0200970 "DMA allocations while released from device "
Stanislaw Gruszkaba4b87ad2011-03-31 08:08:09 -0400971 "[count=%d]\n"
972 "One of leaked entries details: "
973 "[device address=0x%016llx] [size=%llu bytes] "
974 "[mapped with %s] [mapped as %s]\n",
975 count, entry->dev_addr, entry->size,
976 dir2name[entry->direction], type2name[entry->type]);
Joerg Roedeled888ae2009-05-22 17:16:04 +0200977 break;
978 default:
979 break;
980 }
981
982 return 0;
983}
984
Joerg Roedel41531c82009-03-16 17:32:14 +0100985void dma_debug_add_bus(struct bus_type *bus)
986{
Joerg Roedeled888ae2009-05-22 17:16:04 +0200987 struct notifier_block *nb;
988
Florian Fainelli01ce18b2014-12-10 15:41:23 -0800989 if (dma_debug_disabled())
Shaun Ruffellf797d982009-12-17 18:00:36 -0600990 return;
991
Joerg Roedeled888ae2009-05-22 17:16:04 +0200992 nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL);
993 if (nb == NULL) {
Joerg Roedele7ed70e2009-06-08 15:39:24 +0200994 pr_err("dma_debug_add_bus: out of memory\n");
Joerg Roedeled888ae2009-05-22 17:16:04 +0200995 return;
996 }
997
998 nb->notifier_call = dma_debug_device_change;
999
1000 bus_register_notifier(bus, nb);
Joerg Roedel41531c82009-03-16 17:32:14 +01001001}
Joerg Roedel788dcfa2009-01-09 13:13:27 +01001002
Joerg Roedel6bf07872009-01-09 12:54:42 +01001003/*
1004 * Let the architectures decide how many entries should be preallocated.
1005 */
1006void dma_debug_init(u32 num_entries)
1007{
1008 int i;
1009
Florian Fainelli2ce8e7e2014-12-10 15:41:25 -08001010 /* Do not use dma_debug_initialized here, since we really want to be
1011 * called to set dma_debug_initialized
1012 */
1013 if (global_disable)
Joerg Roedel6bf07872009-01-09 12:54:42 +01001014 return;
1015
1016 for (i = 0; i < HASH_SIZE; ++i) {
1017 INIT_LIST_HEAD(&dma_entry_hash[i].list);
Ingo Molnarb0a5b832009-06-16 16:11:14 +02001018 spin_lock_init(&dma_entry_hash[i].lock);
Joerg Roedel6bf07872009-01-09 12:54:42 +01001019 }
1020
Joerg Roedel788dcfa2009-01-09 13:13:27 +01001021 if (dma_debug_fs_init() != 0) {
Joerg Roedele7ed70e2009-06-08 15:39:24 +02001022 pr_err("DMA-API: error creating debugfs entries - disabling\n");
Joerg Roedel788dcfa2009-01-09 13:13:27 +01001023 global_disable = true;
1024
1025 return;
1026 }
1027
Joerg Roedel59d3daa2009-01-09 13:01:56 +01001028 if (req_entries)
1029 num_entries = req_entries;
1030
Joerg Roedel6bf07872009-01-09 12:54:42 +01001031 if (prealloc_memory(num_entries) != 0) {
Joerg Roedele7ed70e2009-06-08 15:39:24 +02001032 pr_err("DMA-API: debugging out of memory error - disabled\n");
Joerg Roedel6bf07872009-01-09 12:54:42 +01001033 global_disable = true;
1034
1035 return;
1036 }
1037
FUJITA Tomonorie6a1a892009-04-15 18:22:41 +09001038 nr_total_entries = num_free_entries;
1039
Florian Fainelli2ce8e7e2014-12-10 15:41:25 -08001040 dma_debug_initialized = true;
1041
Joerg Roedele7ed70e2009-06-08 15:39:24 +02001042 pr_info("DMA-API: debugging enabled by kernel config\n");
Joerg Roedel6bf07872009-01-09 12:54:42 +01001043}
1044
Joerg Roedel59d3daa2009-01-09 13:01:56 +01001045static __init int dma_debug_cmdline(char *str)
1046{
1047 if (!str)
1048 return -EINVAL;
1049
1050 if (strncmp(str, "off", 3) == 0) {
Joerg Roedele7ed70e2009-06-08 15:39:24 +02001051 pr_info("DMA-API: debugging disabled on kernel command line\n");
Joerg Roedel59d3daa2009-01-09 13:01:56 +01001052 global_disable = true;
1053 }
1054
1055 return 0;
1056}
1057
1058static __init int dma_debug_entries_cmdline(char *str)
1059{
1060 int res;
1061
1062 if (!str)
1063 return -EINVAL;
1064
1065 res = get_option(&str, &req_entries);
1066
1067 if (!res)
1068 req_entries = 0;
1069
1070 return 0;
1071}
1072
1073__setup("dma_debug=", dma_debug_cmdline);
1074__setup("dma_debug_entries=", dma_debug_entries_cmdline);
1075
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001076static void check_unmap(struct dma_debug_entry *ref)
1077{
1078 struct dma_debug_entry *entry;
1079 struct hash_bucket *bucket;
1080 unsigned long flags;
1081
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001082 bucket = get_hash_bucket(ref, &flags);
Neil Hormanc6a21d02011-08-08 15:13:54 -04001083 entry = bucket_find_exact(bucket, ref);
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001084
1085 if (!entry) {
Alexander Duyck8d640a52013-03-22 15:04:48 -07001086 /* must drop lock before calling dma_mapping_error */
1087 put_hash_bucket(bucket, &flags);
1088
Shuah Khanbfe0fb02012-11-03 17:00:07 -06001089 if (dma_mapping_error(ref->dev, ref->dev_addr)) {
1090 err_printk(ref->dev, NULL,
Alexander Duyck8d640a52013-03-22 15:04:48 -07001091 "DMA-API: device driver tries to free an "
1092 "invalid DMA memory address\n");
1093 } else {
1094 err_printk(ref->dev, NULL,
1095 "DMA-API: device driver tries to free DMA "
1096 "memory it has not allocated [device "
1097 "address=0x%016llx] [size=%llu bytes]\n",
1098 ref->dev_addr, ref->size);
Shuah Khanbfe0fb02012-11-03 17:00:07 -06001099 }
Alexander Duyck8d640a52013-03-22 15:04:48 -07001100 return;
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001101 }
1102
1103 if (ref->size != entry->size) {
David Woodhouse6c132d12009-01-19 16:52:39 +01001104 err_printk(ref->dev, entry, "DMA-API: device driver frees "
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001105 "DMA memory with different size "
1106 "[device address=0x%016llx] [map size=%llu bytes] "
1107 "[unmap size=%llu bytes]\n",
1108 ref->dev_addr, entry->size, ref->size);
1109 }
1110
1111 if (ref->type != entry->type) {
David Woodhouse6c132d12009-01-19 16:52:39 +01001112 err_printk(ref->dev, entry, "DMA-API: device driver frees "
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001113 "DMA memory with wrong function "
1114 "[device address=0x%016llx] [size=%llu bytes] "
1115 "[mapped as %s] [unmapped as %s]\n",
1116 ref->dev_addr, ref->size,
1117 type2name[entry->type], type2name[ref->type]);
1118 } else if ((entry->type == dma_debug_coherent) &&
Dan Williams0abdd7a2014-01-21 15:48:12 -08001119 (phys_addr(ref) != phys_addr(entry))) {
David Woodhouse6c132d12009-01-19 16:52:39 +01001120 err_printk(ref->dev, entry, "DMA-API: device driver frees "
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001121 "DMA memory with different CPU address "
1122 "[device address=0x%016llx] [size=%llu bytes] "
Joerg Roedel59a40e702009-10-29 16:25:50 +01001123 "[cpu alloc address=0x%016llx] "
1124 "[cpu free address=0x%016llx]",
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001125 ref->dev_addr, ref->size,
Dan Williams0abdd7a2014-01-21 15:48:12 -08001126 phys_addr(entry),
1127 phys_addr(ref));
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001128 }
1129
1130 if (ref->sg_call_ents && ref->type == dma_debug_sg &&
1131 ref->sg_call_ents != entry->sg_call_ents) {
David Woodhouse6c132d12009-01-19 16:52:39 +01001132 err_printk(ref->dev, entry, "DMA-API: device driver frees "
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001133 "DMA sg list with different entry count "
1134 "[map count=%d] [unmap count=%d]\n",
1135 entry->sg_call_ents, ref->sg_call_ents);
1136 }
1137
1138 /*
1139 * This may be no bug in reality - but most implementations of the
1140 * DMA API don't handle this properly, so check for it here
1141 */
1142 if (ref->direction != entry->direction) {
David Woodhouse6c132d12009-01-19 16:52:39 +01001143 err_printk(ref->dev, entry, "DMA-API: device driver frees "
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001144 "DMA memory with different direction "
1145 "[device address=0x%016llx] [size=%llu bytes] "
1146 "[mapped with %s] [unmapped with %s]\n",
1147 ref->dev_addr, ref->size,
1148 dir2name[entry->direction],
1149 dir2name[ref->direction]);
1150 }
1151
Shuah Khan6c9c6d62012-10-08 11:08:06 -06001152 if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {
1153 err_printk(ref->dev, entry,
1154 "DMA-API: device driver failed to check map error"
1155 "[device address=0x%016llx] [size=%llu bytes] "
1156 "[mapped as %s]",
1157 ref->dev_addr, ref->size,
1158 type2name[entry->type]);
1159 }
1160
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001161 hash_bucket_del(entry);
1162 dma_entry_free(entry);
1163
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001164 put_hash_bucket(bucket, &flags);
1165}
1166
1167static void check_for_stack(struct device *dev, void *addr)
1168{
1169 if (object_is_on_stack(addr))
Horia Geantaf9134be2014-09-02 14:28:14 +03001170 err_printk(dev, NULL, "DMA-API: device driver maps memory from "
David Woodhouse6c132d12009-01-19 16:52:39 +01001171 "stack [addr=%p]\n", addr);
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001172}
1173
Ingo Molnarf39d1b92009-07-10 21:38:02 +02001174static inline bool overlap(void *addr, unsigned long len, void *start, void *end)
Joerg Roedel2e34bde2009-03-16 16:51:55 +01001175{
Ingo Molnarf39d1b92009-07-10 21:38:02 +02001176 unsigned long a1 = (unsigned long)addr;
1177 unsigned long b1 = a1 + len;
1178 unsigned long a2 = (unsigned long)start;
1179 unsigned long b2 = (unsigned long)end;
Joerg Roedel2e34bde2009-03-16 16:51:55 +01001180
Ingo Molnarf39d1b92009-07-10 21:38:02 +02001181 return !(b1 <= a2 || a1 >= b2);
Joerg Roedel2e34bde2009-03-16 16:51:55 +01001182}
1183
Ingo Molnarf39d1b92009-07-10 21:38:02 +02001184static void check_for_illegal_area(struct device *dev, void *addr, unsigned long len)
Joerg Roedel2e34bde2009-03-16 16:51:55 +01001185{
Laura Abbottea535e42016-01-14 15:16:50 -08001186 if (overlap(addr, len, _stext, _etext) ||
Ingo Molnarf39d1b92009-07-10 21:38:02 +02001187 overlap(addr, len, __start_rodata, __end_rodata))
1188 err_printk(dev, NULL, "DMA-API: device driver maps memory from kernel text or rodata [addr=%p] [len=%lu]\n", addr, len);
Joerg Roedel2e34bde2009-03-16 16:51:55 +01001189}
1190
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001191static void check_sync(struct device *dev,
1192 struct dma_debug_entry *ref,
1193 bool to_cpu)
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001194{
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001195 struct dma_debug_entry *entry;
1196 struct hash_bucket *bucket;
1197 unsigned long flags;
1198
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001199 bucket = get_hash_bucket(ref, &flags);
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001200
Neil Hormanc6a21d02011-08-08 15:13:54 -04001201 entry = bucket_find_contain(&bucket, ref, &flags);
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001202
1203 if (!entry) {
David Woodhouse6c132d12009-01-19 16:52:39 +01001204 err_printk(dev, NULL, "DMA-API: device driver tries "
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001205 "to sync DMA memory it has not allocated "
1206 "[device address=0x%016llx] [size=%llu bytes]\n",
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001207 (unsigned long long)ref->dev_addr, ref->size);
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001208 goto out;
1209 }
1210
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001211 if (ref->size > entry->size) {
David Woodhouse6c132d12009-01-19 16:52:39 +01001212 err_printk(dev, entry, "DMA-API: device driver syncs"
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001213 " DMA memory outside allocated range "
1214 "[device address=0x%016llx] "
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001215 "[allocation size=%llu bytes] "
1216 "[sync offset+size=%llu]\n",
1217 entry->dev_addr, entry->size,
1218 ref->size);
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001219 }
1220
Krzysztof Halasa42d53b42010-01-08 14:42:36 -08001221 if (entry->direction == DMA_BIDIRECTIONAL)
1222 goto out;
1223
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001224 if (ref->direction != entry->direction) {
David Woodhouse6c132d12009-01-19 16:52:39 +01001225 err_printk(dev, entry, "DMA-API: device driver syncs "
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001226 "DMA memory with different direction "
1227 "[device address=0x%016llx] [size=%llu bytes] "
1228 "[mapped with %s] [synced with %s]\n",
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001229 (unsigned long long)ref->dev_addr, entry->size,
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001230 dir2name[entry->direction],
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001231 dir2name[ref->direction]);
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001232 }
1233
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001234 if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) &&
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001235 !(ref->direction == DMA_TO_DEVICE))
David Woodhouse6c132d12009-01-19 16:52:39 +01001236 err_printk(dev, entry, "DMA-API: device driver syncs "
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001237 "device read-only DMA memory for cpu "
1238 "[device address=0x%016llx] [size=%llu bytes] "
1239 "[mapped with %s] [synced with %s]\n",
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001240 (unsigned long long)ref->dev_addr, entry->size,
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001241 dir2name[entry->direction],
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001242 dir2name[ref->direction]);
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001243
1244 if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) &&
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001245 !(ref->direction == DMA_FROM_DEVICE))
David Woodhouse6c132d12009-01-19 16:52:39 +01001246 err_printk(dev, entry, "DMA-API: device driver syncs "
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001247 "device write-only DMA memory to device "
1248 "[device address=0x%016llx] [size=%llu bytes] "
1249 "[mapped with %s] [synced with %s]\n",
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001250 (unsigned long long)ref->dev_addr, entry->size,
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001251 dir2name[entry->direction],
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001252 dir2name[ref->direction]);
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001253
Robin Murphy7f830642015-11-06 16:32:55 -08001254 if (ref->sg_call_ents && ref->type == dma_debug_sg &&
1255 ref->sg_call_ents != entry->sg_call_ents) {
1256 err_printk(ref->dev, entry, "DMA-API: device driver syncs "
1257 "DMA sg list with different entry count "
1258 "[map count=%d] [sync count=%d]\n",
1259 entry->sg_call_ents, ref->sg_call_ents);
1260 }
1261
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001262out:
1263 put_hash_bucket(bucket, &flags);
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001264}
1265
Joerg Roedelf62bc982009-01-09 14:14:49 +01001266void debug_dma_map_page(struct device *dev, struct page *page, size_t offset,
1267 size_t size, int direction, dma_addr_t dma_addr,
1268 bool map_single)
1269{
1270 struct dma_debug_entry *entry;
1271
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001272 if (unlikely(dma_debug_disabled()))
Joerg Roedelf62bc982009-01-09 14:14:49 +01001273 return;
1274
Shuah Khanbfe0fb02012-11-03 17:00:07 -06001275 if (dma_mapping_error(dev, dma_addr))
Joerg Roedelf62bc982009-01-09 14:14:49 +01001276 return;
1277
1278 entry = dma_entry_alloc();
1279 if (!entry)
1280 return;
1281
1282 entry->dev = dev;
1283 entry->type = dma_debug_page;
Dan Williams0abdd7a2014-01-21 15:48:12 -08001284 entry->pfn = page_to_pfn(page);
1285 entry->offset = offset,
Joerg Roedelf62bc982009-01-09 14:14:49 +01001286 entry->dev_addr = dma_addr;
1287 entry->size = size;
1288 entry->direction = direction;
Shuah Khan6c9c6d62012-10-08 11:08:06 -06001289 entry->map_err_type = MAP_ERR_NOT_CHECKED;
Joerg Roedelf62bc982009-01-09 14:14:49 +01001290
Joerg Roedel9537a482009-03-23 15:35:08 +01001291 if (map_single)
Joerg Roedelf62bc982009-01-09 14:14:49 +01001292 entry->type = dma_debug_single;
Joerg Roedel9537a482009-03-23 15:35:08 +01001293
1294 if (!PageHighMem(page)) {
Ingo Molnarf39d1b92009-07-10 21:38:02 +02001295 void *addr = page_address(page) + offset;
1296
Joerg Roedel2e34bde2009-03-16 16:51:55 +01001297 check_for_stack(dev, addr);
1298 check_for_illegal_area(dev, addr, size);
Joerg Roedelf62bc982009-01-09 14:14:49 +01001299 }
1300
1301 add_dma_entry(entry);
1302}
1303EXPORT_SYMBOL(debug_dma_map_page);
1304
Shuah Khan6c9c6d62012-10-08 11:08:06 -06001305void debug_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
1306{
1307 struct dma_debug_entry ref;
1308 struct dma_debug_entry *entry;
1309 struct hash_bucket *bucket;
1310 unsigned long flags;
1311
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001312 if (unlikely(dma_debug_disabled()))
Shuah Khan6c9c6d62012-10-08 11:08:06 -06001313 return;
1314
1315 ref.dev = dev;
1316 ref.dev_addr = dma_addr;
1317 bucket = get_hash_bucket(&ref, &flags);
Shuah Khan6c9c6d62012-10-08 11:08:06 -06001318
Alexander Duyck96e7d7a2013-03-22 15:04:49 -07001319 list_for_each_entry(entry, &bucket->list, list) {
1320 if (!exact_match(&ref, entry))
1321 continue;
Shuah Khan6c9c6d62012-10-08 11:08:06 -06001322
Alexander Duyck96e7d7a2013-03-22 15:04:49 -07001323 /*
1324 * The same physical address can be mapped multiple
1325 * times. Without a hardware IOMMU this results in the
1326 * same device addresses being put into the dma-debug
1327 * hash multiple times too. This can result in false
1328 * positives being reported. Therefore we implement a
1329 * best-fit algorithm here which updates the first entry
1330 * from the hash which fits the reference value and is
1331 * not currently listed as being checked.
1332 */
1333 if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {
1334 entry->map_err_type = MAP_ERR_CHECKED;
1335 break;
1336 }
1337 }
1338
Shuah Khan6c9c6d62012-10-08 11:08:06 -06001339 put_hash_bucket(bucket, &flags);
1340}
1341EXPORT_SYMBOL(debug_dma_mapping_error);
1342
Joerg Roedelf62bc982009-01-09 14:14:49 +01001343void debug_dma_unmap_page(struct device *dev, dma_addr_t addr,
1344 size_t size, int direction, bool map_single)
1345{
1346 struct dma_debug_entry ref = {
1347 .type = dma_debug_page,
1348 .dev = dev,
1349 .dev_addr = addr,
1350 .size = size,
1351 .direction = direction,
1352 };
1353
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001354 if (unlikely(dma_debug_disabled()))
Joerg Roedelf62bc982009-01-09 14:14:49 +01001355 return;
1356
1357 if (map_single)
1358 ref.type = dma_debug_single;
1359
1360 check_unmap(&ref);
1361}
1362EXPORT_SYMBOL(debug_dma_unmap_page);
1363
Joerg Roedel972aa452009-01-09 14:19:54 +01001364void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
1365 int nents, int mapped_ents, int direction)
1366{
1367 struct dma_debug_entry *entry;
1368 struct scatterlist *s;
1369 int i;
1370
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001371 if (unlikely(dma_debug_disabled()))
Joerg Roedel972aa452009-01-09 14:19:54 +01001372 return;
1373
1374 for_each_sg(sg, s, mapped_ents, i) {
1375 entry = dma_entry_alloc();
1376 if (!entry)
1377 return;
1378
1379 entry->type = dma_debug_sg;
1380 entry->dev = dev;
Dan Williams0abdd7a2014-01-21 15:48:12 -08001381 entry->pfn = page_to_pfn(sg_page(s));
1382 entry->offset = s->offset,
FUJITA Tomonori884d0592009-05-27 09:43:02 +09001383 entry->size = sg_dma_len(s);
FUJITA Tomonori15aedea2009-05-27 09:43:01 +09001384 entry->dev_addr = sg_dma_address(s);
Joerg Roedel972aa452009-01-09 14:19:54 +01001385 entry->direction = direction;
1386 entry->sg_call_ents = nents;
1387 entry->sg_mapped_ents = mapped_ents;
1388
Joerg Roedel9537a482009-03-23 15:35:08 +01001389 if (!PageHighMem(sg_page(s))) {
1390 check_for_stack(dev, sg_virt(s));
FUJITA Tomonori884d0592009-05-27 09:43:02 +09001391 check_for_illegal_area(dev, sg_virt(s), sg_dma_len(s));
Joerg Roedel9537a482009-03-23 15:35:08 +01001392 }
Joerg Roedel972aa452009-01-09 14:19:54 +01001393
1394 add_dma_entry(entry);
1395 }
1396}
1397EXPORT_SYMBOL(debug_dma_map_sg);
1398
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001399static int get_nr_mapped_entries(struct device *dev,
1400 struct dma_debug_entry *ref)
FUJITA Tomonori88f39072009-05-27 09:43:03 +09001401{
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001402 struct dma_debug_entry *entry;
FUJITA Tomonori88f39072009-05-27 09:43:03 +09001403 struct hash_bucket *bucket;
1404 unsigned long flags;
Joerg Roedelc17e2cf2009-06-08 15:19:29 +02001405 int mapped_ents;
FUJITA Tomonori88f39072009-05-27 09:43:03 +09001406
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001407 bucket = get_hash_bucket(ref, &flags);
Neil Hormanc6a21d02011-08-08 15:13:54 -04001408 entry = bucket_find_exact(bucket, ref);
Joerg Roedelc17e2cf2009-06-08 15:19:29 +02001409 mapped_ents = 0;
1410
FUJITA Tomonori88f39072009-05-27 09:43:03 +09001411 if (entry)
1412 mapped_ents = entry->sg_mapped_ents;
1413 put_hash_bucket(bucket, &flags);
1414
1415 return mapped_ents;
1416}
1417
Joerg Roedel972aa452009-01-09 14:19:54 +01001418void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
1419 int nelems, int dir)
1420{
Joerg Roedel972aa452009-01-09 14:19:54 +01001421 struct scatterlist *s;
1422 int mapped_ents = 0, i;
Joerg Roedel972aa452009-01-09 14:19:54 +01001423
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001424 if (unlikely(dma_debug_disabled()))
Joerg Roedel972aa452009-01-09 14:19:54 +01001425 return;
1426
1427 for_each_sg(sglist, s, nelems, i) {
1428
1429 struct dma_debug_entry ref = {
1430 .type = dma_debug_sg,
1431 .dev = dev,
Dan Williams0abdd7a2014-01-21 15:48:12 -08001432 .pfn = page_to_pfn(sg_page(s)),
1433 .offset = s->offset,
FUJITA Tomonori15aedea2009-05-27 09:43:01 +09001434 .dev_addr = sg_dma_address(s),
FUJITA Tomonori884d0592009-05-27 09:43:02 +09001435 .size = sg_dma_len(s),
Joerg Roedel972aa452009-01-09 14:19:54 +01001436 .direction = dir,
Joerg Roedele5e8c5b2009-06-11 10:03:42 +02001437 .sg_call_ents = nelems,
Joerg Roedel972aa452009-01-09 14:19:54 +01001438 };
1439
1440 if (mapped_ents && i >= mapped_ents)
1441 break;
1442
Joerg Roedele5e8c5b2009-06-11 10:03:42 +02001443 if (!i)
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001444 mapped_ents = get_nr_mapped_entries(dev, &ref);
Joerg Roedel972aa452009-01-09 14:19:54 +01001445
1446 check_unmap(&ref);
1447 }
1448}
1449EXPORT_SYMBOL(debug_dma_unmap_sg);
1450
Joerg Roedel6bfd4492009-01-09 14:38:50 +01001451void debug_dma_alloc_coherent(struct device *dev, size_t size,
1452 dma_addr_t dma_addr, void *virt)
1453{
1454 struct dma_debug_entry *entry;
1455
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001456 if (unlikely(dma_debug_disabled()))
Joerg Roedel6bfd4492009-01-09 14:38:50 +01001457 return;
1458
1459 if (unlikely(virt == NULL))
1460 return;
1461
1462 entry = dma_entry_alloc();
1463 if (!entry)
1464 return;
1465
1466 entry->type = dma_debug_coherent;
1467 entry->dev = dev;
Dan Williams0abdd7a2014-01-21 15:48:12 -08001468 entry->pfn = page_to_pfn(virt_to_page(virt));
Daniel Mentz0354aec2015-12-15 17:38:48 -08001469 entry->offset = (size_t) virt & ~PAGE_MASK;
Joerg Roedel6bfd4492009-01-09 14:38:50 +01001470 entry->size = size;
1471 entry->dev_addr = dma_addr;
1472 entry->direction = DMA_BIDIRECTIONAL;
1473
1474 add_dma_entry(entry);
1475}
1476EXPORT_SYMBOL(debug_dma_alloc_coherent);
1477
1478void debug_dma_free_coherent(struct device *dev, size_t size,
1479 void *virt, dma_addr_t addr)
1480{
1481 struct dma_debug_entry ref = {
1482 .type = dma_debug_coherent,
1483 .dev = dev,
Dan Williams0abdd7a2014-01-21 15:48:12 -08001484 .pfn = page_to_pfn(virt_to_page(virt)),
Daniel Mentz0354aec2015-12-15 17:38:48 -08001485 .offset = (size_t) virt & ~PAGE_MASK,
Joerg Roedel6bfd4492009-01-09 14:38:50 +01001486 .dev_addr = addr,
1487 .size = size,
1488 .direction = DMA_BIDIRECTIONAL,
1489 };
1490
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001491 if (unlikely(dma_debug_disabled()))
Joerg Roedel6bfd4492009-01-09 14:38:50 +01001492 return;
1493
1494 check_unmap(&ref);
1495}
1496EXPORT_SYMBOL(debug_dma_free_coherent);
1497
Joerg Roedelb9d23172009-01-09 14:43:04 +01001498void debug_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
1499 size_t size, int direction)
1500{
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001501 struct dma_debug_entry ref;
1502
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001503 if (unlikely(dma_debug_disabled()))
Joerg Roedelb9d23172009-01-09 14:43:04 +01001504 return;
1505
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001506 ref.type = dma_debug_single;
1507 ref.dev = dev;
1508 ref.dev_addr = dma_handle;
1509 ref.size = size;
1510 ref.direction = direction;
1511 ref.sg_call_ents = 0;
1512
1513 check_sync(dev, &ref, true);
Joerg Roedelb9d23172009-01-09 14:43:04 +01001514}
1515EXPORT_SYMBOL(debug_dma_sync_single_for_cpu);
1516
1517void debug_dma_sync_single_for_device(struct device *dev,
1518 dma_addr_t dma_handle, size_t size,
1519 int direction)
1520{
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001521 struct dma_debug_entry ref;
1522
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001523 if (unlikely(dma_debug_disabled()))
Joerg Roedelb9d23172009-01-09 14:43:04 +01001524 return;
1525
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001526 ref.type = dma_debug_single;
1527 ref.dev = dev;
1528 ref.dev_addr = dma_handle;
1529 ref.size = size;
1530 ref.direction = direction;
1531 ref.sg_call_ents = 0;
1532
1533 check_sync(dev, &ref, false);
Joerg Roedelb9d23172009-01-09 14:43:04 +01001534}
1535EXPORT_SYMBOL(debug_dma_sync_single_for_device);
1536
Joerg Roedel948408b2009-01-09 14:55:38 +01001537void debug_dma_sync_single_range_for_cpu(struct device *dev,
1538 dma_addr_t dma_handle,
1539 unsigned long offset, size_t size,
1540 int direction)
1541{
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001542 struct dma_debug_entry ref;
1543
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001544 if (unlikely(dma_debug_disabled()))
Joerg Roedel948408b2009-01-09 14:55:38 +01001545 return;
1546
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001547 ref.type = dma_debug_single;
1548 ref.dev = dev;
1549 ref.dev_addr = dma_handle;
1550 ref.size = offset + size;
1551 ref.direction = direction;
1552 ref.sg_call_ents = 0;
1553
1554 check_sync(dev, &ref, true);
Joerg Roedel948408b2009-01-09 14:55:38 +01001555}
1556EXPORT_SYMBOL(debug_dma_sync_single_range_for_cpu);
1557
1558void debug_dma_sync_single_range_for_device(struct device *dev,
1559 dma_addr_t dma_handle,
1560 unsigned long offset,
1561 size_t size, int direction)
1562{
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001563 struct dma_debug_entry ref;
1564
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001565 if (unlikely(dma_debug_disabled()))
Joerg Roedel948408b2009-01-09 14:55:38 +01001566 return;
1567
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001568 ref.type = dma_debug_single;
1569 ref.dev = dev;
1570 ref.dev_addr = dma_handle;
1571 ref.size = offset + size;
1572 ref.direction = direction;
1573 ref.sg_call_ents = 0;
1574
1575 check_sync(dev, &ref, false);
Joerg Roedel948408b2009-01-09 14:55:38 +01001576}
1577EXPORT_SYMBOL(debug_dma_sync_single_range_for_device);
1578
Joerg Roedela31fba52009-01-09 15:01:12 +01001579void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1580 int nelems, int direction)
1581{
1582 struct scatterlist *s;
FUJITA Tomonori88f39072009-05-27 09:43:03 +09001583 int mapped_ents = 0, i;
Joerg Roedela31fba52009-01-09 15:01:12 +01001584
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001585 if (unlikely(dma_debug_disabled()))
Joerg Roedela31fba52009-01-09 15:01:12 +01001586 return;
1587
1588 for_each_sg(sg, s, nelems, i) {
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001589
1590 struct dma_debug_entry ref = {
1591 .type = dma_debug_sg,
1592 .dev = dev,
Dan Williams0abdd7a2014-01-21 15:48:12 -08001593 .pfn = page_to_pfn(sg_page(s)),
1594 .offset = s->offset,
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001595 .dev_addr = sg_dma_address(s),
1596 .size = sg_dma_len(s),
1597 .direction = direction,
1598 .sg_call_ents = nelems,
1599 };
1600
FUJITA Tomonori88f39072009-05-27 09:43:03 +09001601 if (!i)
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001602 mapped_ents = get_nr_mapped_entries(dev, &ref);
FUJITA Tomonori88f39072009-05-27 09:43:03 +09001603
1604 if (i >= mapped_ents)
1605 break;
1606
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001607 check_sync(dev, &ref, true);
Joerg Roedela31fba52009-01-09 15:01:12 +01001608 }
1609}
1610EXPORT_SYMBOL(debug_dma_sync_sg_for_cpu);
1611
1612void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1613 int nelems, int direction)
1614{
1615 struct scatterlist *s;
FUJITA Tomonori88f39072009-05-27 09:43:03 +09001616 int mapped_ents = 0, i;
Joerg Roedela31fba52009-01-09 15:01:12 +01001617
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001618 if (unlikely(dma_debug_disabled()))
Joerg Roedela31fba52009-01-09 15:01:12 +01001619 return;
1620
1621 for_each_sg(sg, s, nelems, i) {
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001622
1623 struct dma_debug_entry ref = {
1624 .type = dma_debug_sg,
1625 .dev = dev,
Dan Williams0abdd7a2014-01-21 15:48:12 -08001626 .pfn = page_to_pfn(sg_page(s)),
1627 .offset = s->offset,
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001628 .dev_addr = sg_dma_address(s),
1629 .size = sg_dma_len(s),
1630 .direction = direction,
1631 .sg_call_ents = nelems,
1632 };
FUJITA Tomonori88f39072009-05-27 09:43:03 +09001633 if (!i)
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001634 mapped_ents = get_nr_mapped_entries(dev, &ref);
FUJITA Tomonori88f39072009-05-27 09:43:03 +09001635
1636 if (i >= mapped_ents)
1637 break;
1638
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001639 check_sync(dev, &ref, false);
Joerg Roedela31fba52009-01-09 15:01:12 +01001640 }
1641}
1642EXPORT_SYMBOL(debug_dma_sync_sg_for_device);
1643
Joerg Roedel1745de52009-05-22 21:49:51 +02001644static int __init dma_debug_driver_setup(char *str)
1645{
1646 int i;
1647
1648 for (i = 0; i < NAME_MAX_LEN - 1; ++i, ++str) {
1649 current_driver_name[i] = *str;
1650 if (*str == 0)
1651 break;
1652 }
1653
1654 if (current_driver_name[0])
Joerg Roedele7ed70e2009-06-08 15:39:24 +02001655 pr_info("DMA-API: enable driver filter for driver [%s]\n",
1656 current_driver_name);
Joerg Roedel1745de52009-05-22 21:49:51 +02001657
1658
1659 return 1;
1660}
1661__setup("dma_debug_driver=", dma_debug_driver_setup);