blob: ea4cc3dde4f1bac9f3b8337fdac95408cb3e8b35 [file] [log] [blame]
Joerg Roedelf2f45e52009-01-09 12:19:52 +01001/*
2 * Copyright (C) 2008 Advanced Micro Devices, Inc.
3 *
4 * Author: Joerg Roedel <joerg.roedel@amd.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
Ingo Molnar68db0cf2017-02-08 18:51:37 +010020#include <linux/sched/task_stack.h>
Joerg Roedel972aa452009-01-09 14:19:54 +010021#include <linux/scatterlist.h>
Joerg Roedel2d62ece2009-01-09 14:10:26 +010022#include <linux/dma-mapping.h>
Ingo Molnar29930022017-02-08 18:51:36 +010023#include <linux/sched/task.h>
David Woodhouse6c132d12009-01-19 16:52:39 +010024#include <linux/stacktrace.h>
Joerg Roedelf2f45e52009-01-09 12:19:52 +010025#include <linux/dma-debug.h>
Joerg Roedel30dfa902009-01-09 12:34:49 +010026#include <linux/spinlock.h>
Andy Lutomirskib4a0f532016-08-11 02:35:22 -070027#include <linux/vmalloc.h>
Joerg Roedel788dcfa2009-01-09 13:13:27 +010028#include <linux/debugfs.h>
Joerg Roedel8a6fc702009-05-22 21:23:13 +020029#include <linux/uaccess.h>
Paul Gortmaker23a7bfa2011-07-01 16:23:59 -040030#include <linux/export.h>
Joerg Roedel2d62ece2009-01-09 14:10:26 +010031#include <linux/device.h>
Joerg Roedelf2f45e52009-01-09 12:19:52 +010032#include <linux/types.h>
Joerg Roedel2d62ece2009-01-09 14:10:26 +010033#include <linux/sched.h>
Joerg Roedel8a6fc702009-05-22 21:23:13 +020034#include <linux/ctype.h>
Joerg Roedelf2f45e52009-01-09 12:19:52 +010035#include <linux/list.h>
Joerg Roedel6bf07872009-01-09 12:54:42 +010036#include <linux/slab.h>
Joerg Roedelf2f45e52009-01-09 12:19:52 +010037
Joerg Roedel2e34bde2009-03-16 16:51:55 +010038#include <asm/sections.h>
39
Joerg Roedel30dfa902009-01-09 12:34:49 +010040#define HASH_SIZE 1024ULL
41#define HASH_FN_SHIFT 13
42#define HASH_FN_MASK (HASH_SIZE - 1)
43
Joerg Roedelf2f45e52009-01-09 12:19:52 +010044enum {
45 dma_debug_single,
46 dma_debug_page,
47 dma_debug_sg,
48 dma_debug_coherent,
Niklas Söderlund0e74b342016-08-10 13:22:15 +020049 dma_debug_resource,
Joerg Roedelf2f45e52009-01-09 12:19:52 +010050};
51
Shuah Khan6c9c6d62012-10-08 11:08:06 -060052enum map_err_types {
53 MAP_ERR_CHECK_NOT_APPLICABLE,
54 MAP_ERR_NOT_CHECKED,
55 MAP_ERR_CHECKED,
56};
57
David Woodhouse6c132d12009-01-19 16:52:39 +010058#define DMA_DEBUG_STACKTRACE_ENTRIES 5
59
Dan Williams0abdd7a2014-01-21 15:48:12 -080060/**
61 * struct dma_debug_entry - track a dma_map* or dma_alloc_coherent mapping
62 * @list: node on pre-allocated free_entries list
63 * @dev: 'dev' argument to dma_map_{page|single|sg} or dma_alloc_coherent
64 * @type: single, page, sg, coherent
65 * @pfn: page frame of the start address
66 * @offset: offset of mapping relative to pfn
67 * @size: length of the mapping
68 * @direction: enum dma_data_direction
69 * @sg_call_ents: 'nents' from dma_map_sg
70 * @sg_mapped_ents: 'mapped_ents' from dma_map_sg
71 * @map_err_type: track whether dma_mapping_error() was checked
72 * @stacktrace: support backtraces when a violation is detected
73 */
Joerg Roedelf2f45e52009-01-09 12:19:52 +010074struct dma_debug_entry {
75 struct list_head list;
76 struct device *dev;
77 int type;
Dan Williams0abdd7a2014-01-21 15:48:12 -080078 unsigned long pfn;
79 size_t offset;
Joerg Roedelf2f45e52009-01-09 12:19:52 +010080 u64 dev_addr;
81 u64 size;
82 int direction;
83 int sg_call_ents;
84 int sg_mapped_ents;
Shuah Khan6c9c6d62012-10-08 11:08:06 -060085 enum map_err_types map_err_type;
David Woodhouse6c132d12009-01-19 16:52:39 +010086#ifdef CONFIG_STACKTRACE
87 struct stack_trace stacktrace;
88 unsigned long st_entries[DMA_DEBUG_STACKTRACE_ENTRIES];
89#endif
Joerg Roedelf2f45e52009-01-09 12:19:52 +010090};
91
Neil Hormanc6a21d02011-08-08 15:13:54 -040092typedef bool (*match_fn)(struct dma_debug_entry *, struct dma_debug_entry *);
93
Joerg Roedel30dfa902009-01-09 12:34:49 +010094struct hash_bucket {
95 struct list_head list;
96 spinlock_t lock;
Joerg Roedel2d62ece2009-01-09 14:10:26 +010097} ____cacheline_aligned_in_smp;
Joerg Roedel30dfa902009-01-09 12:34:49 +010098
99/* Hash list to save the allocated dma addresses */
100static struct hash_bucket dma_entry_hash[HASH_SIZE];
Joerg Roedel3b1e79e2009-01-09 12:42:46 +0100101/* List of pre-allocated dma_debug_entry's */
102static LIST_HEAD(free_entries);
103/* Lock for the list above */
104static DEFINE_SPINLOCK(free_entries_lock);
105
106/* Global disable flag - will be set in case of an error */
Viresh Kumar621a5f72015-09-26 15:04:07 -0700107static bool global_disable __read_mostly;
Joerg Roedel3b1e79e2009-01-09 12:42:46 +0100108
Florian Fainelli2ce8e7e2014-12-10 15:41:25 -0800109/* Early initialization disable flag, set at the end of dma_debug_init */
110static bool dma_debug_initialized __read_mostly;
111
Florian Fainelli01ce18b2014-12-10 15:41:23 -0800112static inline bool dma_debug_disabled(void)
113{
Florian Fainelli2ce8e7e2014-12-10 15:41:25 -0800114 return global_disable || !dma_debug_initialized;
Florian Fainelli01ce18b2014-12-10 15:41:23 -0800115}
116
Joerg Roedel788dcfa2009-01-09 13:13:27 +0100117/* Global error count */
118static u32 error_count;
119
120/* Global error show enable*/
121static u32 show_all_errors __read_mostly;
122/* Number of errors to show */
123static u32 show_num_errors = 1;
124
Joerg Roedel3b1e79e2009-01-09 12:42:46 +0100125static u32 num_free_entries;
126static u32 min_free_entries;
FUJITA Tomonorie6a1a892009-04-15 18:22:41 +0900127static u32 nr_total_entries;
Joerg Roedel30dfa902009-01-09 12:34:49 +0100128
Joerg Roedel59d3daa2009-01-09 13:01:56 +0100129/* number of preallocated entries requested by kernel cmdline */
130static u32 req_entries;
131
Joerg Roedel788dcfa2009-01-09 13:13:27 +0100132/* debugfs dentry's for the stuff above */
133static struct dentry *dma_debug_dent __read_mostly;
134static struct dentry *global_disable_dent __read_mostly;
135static struct dentry *error_count_dent __read_mostly;
136static struct dentry *show_all_errors_dent __read_mostly;
137static struct dentry *show_num_errors_dent __read_mostly;
138static struct dentry *num_free_entries_dent __read_mostly;
139static struct dentry *min_free_entries_dent __read_mostly;
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200140static struct dentry *filter_dent __read_mostly;
Joerg Roedel788dcfa2009-01-09 13:13:27 +0100141
Joerg Roedel2e507d82009-05-22 18:24:20 +0200142/* per-driver filter related state */
143
144#define NAME_MAX_LEN 64
145
146static char current_driver_name[NAME_MAX_LEN] __read_mostly;
147static struct device_driver *current_driver __read_mostly;
148
149static DEFINE_RWLOCK(driver_name_lock);
Joerg Roedel30dfa902009-01-09 12:34:49 +0100150
Shuah Khan6c9c6d62012-10-08 11:08:06 -0600151static const char *const maperr2str[] = {
152 [MAP_ERR_CHECK_NOT_APPLICABLE] = "dma map error check not applicable",
153 [MAP_ERR_NOT_CHECKED] = "dma map error not checked",
154 [MAP_ERR_CHECKED] = "dma map error checked",
155};
156
Niklas Söderlund0e74b342016-08-10 13:22:15 +0200157static const char *type2name[5] = { "single", "page",
158 "scather-gather", "coherent",
159 "resource" };
Joerg Roedel2d62ece2009-01-09 14:10:26 +0100160
161static const char *dir2name[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE",
162 "DMA_FROM_DEVICE", "DMA_NONE" };
163
164/*
165 * The access to some variables in this macro is racy. We can't use atomic_t
166 * here because all these variables are exported to debugfs. Some of them even
167 * writeable. This is also the reason why a lock won't help much. But anyway,
168 * the races are no big deal. Here is why:
169 *
170 * error_count: the addition is racy, but the worst thing that can happen is
171 * that we don't count some errors
172 * show_num_errors: the subtraction is racy. Also no big deal because in
173 * worst case this will result in one warning more in the
174 * system log than the user configured. This variable is
175 * writeable via debugfs.
176 */
David Woodhouse6c132d12009-01-19 16:52:39 +0100177static inline void dump_entry_trace(struct dma_debug_entry *entry)
178{
179#ifdef CONFIG_STACKTRACE
180 if (entry) {
Joerg Roedele7ed70e2009-06-08 15:39:24 +0200181 pr_warning("Mapped at:\n");
David Woodhouse6c132d12009-01-19 16:52:39 +0100182 print_stack_trace(&entry->stacktrace, 0);
183 }
184#endif
185}
186
Joerg Roedel2e507d82009-05-22 18:24:20 +0200187static bool driver_filter(struct device *dev)
188{
Joerg Roedel0bf84122009-06-08 15:53:46 +0200189 struct device_driver *drv;
190 unsigned long flags;
191 bool ret;
192
Joerg Roedel2e507d82009-05-22 18:24:20 +0200193 /* driver filter off */
194 if (likely(!current_driver_name[0]))
195 return true;
196
197 /* driver filter on and initialized */
Kyle McMartinec9c96e2009-08-19 21:17:08 -0400198 if (current_driver && dev && dev->driver == current_driver)
Joerg Roedel2e507d82009-05-22 18:24:20 +0200199 return true;
200
Kyle McMartinec9c96e2009-08-19 21:17:08 -0400201 /* driver filter on, but we can't filter on a NULL device... */
202 if (!dev)
203 return false;
204
Joerg Roedel0bf84122009-06-08 15:53:46 +0200205 if (current_driver || !current_driver_name[0])
206 return false;
207
Joerg Roedel2e507d82009-05-22 18:24:20 +0200208 /* driver filter on but not yet initialized */
Alan Sternf3ff9242012-01-24 13:35:24 -0500209 drv = dev->driver;
Joerg Roedel0bf84122009-06-08 15:53:46 +0200210 if (!drv)
211 return false;
Joerg Roedel2e507d82009-05-22 18:24:20 +0200212
Joerg Roedel0bf84122009-06-08 15:53:46 +0200213 /* lock to protect against change of current_driver_name */
214 read_lock_irqsave(&driver_name_lock, flags);
Joerg Roedel2e507d82009-05-22 18:24:20 +0200215
Joerg Roedel0bf84122009-06-08 15:53:46 +0200216 ret = false;
217 if (drv->name &&
218 strncmp(current_driver_name, drv->name, NAME_MAX_LEN - 1) == 0) {
219 current_driver = drv;
220 ret = true;
Joerg Roedel2e507d82009-05-22 18:24:20 +0200221 }
222
Joerg Roedel0bf84122009-06-08 15:53:46 +0200223 read_unlock_irqrestore(&driver_name_lock, flags);
Joerg Roedel0bf84122009-06-08 15:53:46 +0200224
225 return ret;
Joerg Roedel2e507d82009-05-22 18:24:20 +0200226}
227
Kyle McMartinec9c96e2009-08-19 21:17:08 -0400228#define err_printk(dev, entry, format, arg...) do { \
229 error_count += 1; \
230 if (driver_filter(dev) && \
231 (show_all_errors || show_num_errors > 0)) { \
232 WARN(1, "%s %s: " format, \
233 dev ? dev_driver_string(dev) : "NULL", \
234 dev ? dev_name(dev) : "NULL", ## arg); \
235 dump_entry_trace(entry); \
236 } \
237 if (!show_all_errors && show_num_errors > 0) \
238 show_num_errors -= 1; \
Joerg Roedel2d62ece2009-01-09 14:10:26 +0100239 } while (0);
240
Joerg Roedel30dfa902009-01-09 12:34:49 +0100241/*
242 * Hash related functions
243 *
244 * Every DMA-API request is saved into a struct dma_debug_entry. To
245 * have quick access to these structs they are stored into a hash.
246 */
247static int hash_fn(struct dma_debug_entry *entry)
248{
249 /*
250 * Hash function is based on the dma address.
251 * We use bits 20-27 here as the index into the hash
252 */
253 return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK;
254}
255
256/*
257 * Request exclusive access to a hash bucket for a given dma_debug_entry.
258 */
259static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry,
260 unsigned long *flags)
Stephen Boydd5dfc802016-07-26 15:21:08 -0700261 __acquires(&dma_entry_hash[idx].lock)
Joerg Roedel30dfa902009-01-09 12:34:49 +0100262{
263 int idx = hash_fn(entry);
264 unsigned long __flags;
265
266 spin_lock_irqsave(&dma_entry_hash[idx].lock, __flags);
267 *flags = __flags;
268 return &dma_entry_hash[idx];
269}
270
271/*
272 * Give up exclusive access to the hash bucket
273 */
274static void put_hash_bucket(struct hash_bucket *bucket,
275 unsigned long *flags)
Stephen Boydd5dfc802016-07-26 15:21:08 -0700276 __releases(&bucket->lock)
Joerg Roedel30dfa902009-01-09 12:34:49 +0100277{
278 unsigned long __flags = *flags;
279
280 spin_unlock_irqrestore(&bucket->lock, __flags);
281}
282
Neil Hormanc6a21d02011-08-08 15:13:54 -0400283static bool exact_match(struct dma_debug_entry *a, struct dma_debug_entry *b)
284{
Thomas Jarosch91ec37c2011-11-17 20:31:02 +0100285 return ((a->dev_addr == b->dev_addr) &&
Neil Hormanc6a21d02011-08-08 15:13:54 -0400286 (a->dev == b->dev)) ? true : false;
287}
288
289static bool containing_match(struct dma_debug_entry *a,
290 struct dma_debug_entry *b)
291{
292 if (a->dev != b->dev)
293 return false;
294
295 if ((b->dev_addr <= a->dev_addr) &&
296 ((b->dev_addr + b->size) >= (a->dev_addr + a->size)))
297 return true;
298
299 return false;
300}
301
Joerg Roedel30dfa902009-01-09 12:34:49 +0100302/*
303 * Search a given entry in the hash bucket list
304 */
Neil Hormanc6a21d02011-08-08 15:13:54 -0400305static struct dma_debug_entry *__hash_bucket_find(struct hash_bucket *bucket,
306 struct dma_debug_entry *ref,
307 match_fn match)
Joerg Roedel30dfa902009-01-09 12:34:49 +0100308{
Joerg Roedel7caf6a42009-06-05 12:01:35 +0200309 struct dma_debug_entry *entry, *ret = NULL;
Ming Leife73fbe2012-10-19 13:57:01 -0700310 int matches = 0, match_lvl, last_lvl = -1;
Joerg Roedel30dfa902009-01-09 12:34:49 +0100311
312 list_for_each_entry(entry, &bucket->list, list) {
Neil Hormanc6a21d02011-08-08 15:13:54 -0400313 if (!match(ref, entry))
Joerg Roedel7caf6a42009-06-05 12:01:35 +0200314 continue;
315
316 /*
317 * Some drivers map the same physical address multiple
318 * times. Without a hardware IOMMU this results in the
319 * same device addresses being put into the dma-debug
320 * hash multiple times too. This can result in false
André Goddard Rosaaf901ca2009-11-14 13:09:05 -0200321 * positives being reported. Therefore we implement a
Joerg Roedel7caf6a42009-06-05 12:01:35 +0200322 * best-fit algorithm here which returns the entry from
323 * the hash which fits best to the reference value
324 * instead of the first-fit.
325 */
326 matches += 1;
327 match_lvl = 0;
Joerg Roedele5e8c5b2009-06-11 10:03:42 +0200328 entry->size == ref->size ? ++match_lvl : 0;
329 entry->type == ref->type ? ++match_lvl : 0;
330 entry->direction == ref->direction ? ++match_lvl : 0;
331 entry->sg_call_ents == ref->sg_call_ents ? ++match_lvl : 0;
Joerg Roedel7caf6a42009-06-05 12:01:35 +0200332
Joerg Roedele5e8c5b2009-06-11 10:03:42 +0200333 if (match_lvl == 4) {
Joerg Roedel7caf6a42009-06-05 12:01:35 +0200334 /* perfect-fit - return the result */
Joerg Roedel30dfa902009-01-09 12:34:49 +0100335 return entry;
Joerg Roedel7caf6a42009-06-05 12:01:35 +0200336 } else if (match_lvl > last_lvl) {
337 /*
338 * We found an entry that fits better then the
Ming Leife73fbe2012-10-19 13:57:01 -0700339 * previous one or it is the 1st match.
Joerg Roedel7caf6a42009-06-05 12:01:35 +0200340 */
341 last_lvl = match_lvl;
342 ret = entry;
343 }
Joerg Roedel30dfa902009-01-09 12:34:49 +0100344 }
345
Joerg Roedel7caf6a42009-06-05 12:01:35 +0200346 /*
347 * If we have multiple matches but no perfect-fit, just return
348 * NULL.
349 */
350 ret = (matches == 1) ? ret : NULL;
351
352 return ret;
Joerg Roedel30dfa902009-01-09 12:34:49 +0100353}
354
Neil Hormanc6a21d02011-08-08 15:13:54 -0400355static struct dma_debug_entry *bucket_find_exact(struct hash_bucket *bucket,
356 struct dma_debug_entry *ref)
357{
358 return __hash_bucket_find(bucket, ref, exact_match);
359}
360
361static struct dma_debug_entry *bucket_find_contain(struct hash_bucket **bucket,
362 struct dma_debug_entry *ref,
363 unsigned long *flags)
364{
365
366 unsigned int max_range = dma_get_max_seg_size(ref->dev);
367 struct dma_debug_entry *entry, index = *ref;
368 unsigned int range = 0;
369
370 while (range <= max_range) {
Sebastian Otta7a2c02a42015-04-16 12:43:25 -0700371 entry = __hash_bucket_find(*bucket, ref, containing_match);
Neil Hormanc6a21d02011-08-08 15:13:54 -0400372
373 if (entry)
374 return entry;
375
376 /*
377 * Nothing found, go back a hash bucket
378 */
379 put_hash_bucket(*bucket, flags);
380 range += (1 << HASH_FN_SHIFT);
381 index.dev_addr -= (1 << HASH_FN_SHIFT);
382 *bucket = get_hash_bucket(&index, flags);
383 }
384
385 return NULL;
386}
387
Joerg Roedel30dfa902009-01-09 12:34:49 +0100388/*
389 * Add an entry to a hash bucket
390 */
391static void hash_bucket_add(struct hash_bucket *bucket,
392 struct dma_debug_entry *entry)
393{
394 list_add_tail(&entry->list, &bucket->list);
395}
396
397/*
398 * Remove entry from a hash bucket list
399 */
400static void hash_bucket_del(struct dma_debug_entry *entry)
401{
402 list_del(&entry->list);
403}
404
Dan Williams0abdd7a2014-01-21 15:48:12 -0800405static unsigned long long phys_addr(struct dma_debug_entry *entry)
406{
Niklas Söderlund0e74b342016-08-10 13:22:15 +0200407 if (entry->type == dma_debug_resource)
408 return __pfn_to_phys(entry->pfn) + entry->offset;
409
Dan Williams0abdd7a2014-01-21 15:48:12 -0800410 return page_to_phys(pfn_to_page(entry->pfn)) + entry->offset;
411}
412
Joerg Roedel30dfa902009-01-09 12:34:49 +0100413/*
David Woodhouseac26c182009-02-12 16:19:13 +0100414 * Dump mapping entries for debugging purposes
415 */
416void debug_dma_dump_mappings(struct device *dev)
417{
418 int idx;
419
420 for (idx = 0; idx < HASH_SIZE; idx++) {
421 struct hash_bucket *bucket = &dma_entry_hash[idx];
422 struct dma_debug_entry *entry;
423 unsigned long flags;
424
425 spin_lock_irqsave(&bucket->lock, flags);
426
427 list_for_each_entry(entry, &bucket->list, list) {
428 if (!dev || dev == entry->dev) {
429 dev_info(entry->dev,
Dan Williams0abdd7a2014-01-21 15:48:12 -0800430 "%s idx %d P=%Lx N=%lx D=%Lx L=%Lx %s %s\n",
David Woodhouseac26c182009-02-12 16:19:13 +0100431 type2name[entry->type], idx,
Dan Williams0abdd7a2014-01-21 15:48:12 -0800432 phys_addr(entry), entry->pfn,
David Woodhouseac26c182009-02-12 16:19:13 +0100433 entry->dev_addr, entry->size,
Shuah Khan6c9c6d62012-10-08 11:08:06 -0600434 dir2name[entry->direction],
435 maperr2str[entry->map_err_type]);
David Woodhouseac26c182009-02-12 16:19:13 +0100436 }
437 }
438
439 spin_unlock_irqrestore(&bucket->lock, flags);
440 }
441}
442EXPORT_SYMBOL(debug_dma_dump_mappings);
443
444/*
Dan Williams3b7a6412014-03-03 15:38:21 -0800445 * For each mapping (initial cacheline in the case of
446 * dma_alloc_coherent/dma_map_page, initial cacheline in each page of a
447 * scatterlist, or the cacheline specified in dma_map_single) insert
448 * into this tree using the cacheline as the key. At
Dan Williams0abdd7a2014-01-21 15:48:12 -0800449 * dma_unmap_{single|sg|page} or dma_free_coherent delete the entry. If
Dan Williams3b7a6412014-03-03 15:38:21 -0800450 * the entry already exists at insertion time add a tag as a reference
Dan Williams0abdd7a2014-01-21 15:48:12 -0800451 * count for the overlapping mappings. For now, the overlap tracking
Dan Williams3b7a6412014-03-03 15:38:21 -0800452 * just ensures that 'unmaps' balance 'maps' before marking the
453 * cacheline idle, but we should also be flagging overlaps as an API
454 * violation.
Dan Williams0abdd7a2014-01-21 15:48:12 -0800455 *
456 * Memory usage is mostly constrained by the maximum number of available
457 * dma-debug entries in that we need a free dma_debug_entry before
Dan Williams3b7a6412014-03-03 15:38:21 -0800458 * inserting into the tree. In the case of dma_map_page and
459 * dma_alloc_coherent there is only one dma_debug_entry and one
460 * dma_active_cacheline entry to track per event. dma_map_sg(), on the
461 * other hand, consumes a single dma_debug_entry, but inserts 'nents'
462 * entries into the tree.
Dan Williams0abdd7a2014-01-21 15:48:12 -0800463 *
464 * At any time debug_dma_assert_idle() can be called to trigger a
Dan Williams3b7a6412014-03-03 15:38:21 -0800465 * warning if any cachelines in the given page are in the active set.
Dan Williams0abdd7a2014-01-21 15:48:12 -0800466 */
Dan Williams3b7a6412014-03-03 15:38:21 -0800467static RADIX_TREE(dma_active_cacheline, GFP_NOWAIT);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800468static DEFINE_SPINLOCK(radix_lock);
Dan Williams3b7a6412014-03-03 15:38:21 -0800469#define ACTIVE_CACHELINE_MAX_OVERLAP ((1 << RADIX_TREE_MAX_TAGS) - 1)
470#define CACHELINE_PER_PAGE_SHIFT (PAGE_SHIFT - L1_CACHE_SHIFT)
471#define CACHELINES_PER_PAGE (1 << CACHELINE_PER_PAGE_SHIFT)
Dan Williams0abdd7a2014-01-21 15:48:12 -0800472
Dan Williams3b7a6412014-03-03 15:38:21 -0800473static phys_addr_t to_cacheline_number(struct dma_debug_entry *entry)
474{
475 return (entry->pfn << CACHELINE_PER_PAGE_SHIFT) +
476 (entry->offset >> L1_CACHE_SHIFT);
477}
478
479static int active_cacheline_read_overlap(phys_addr_t cln)
Dan Williams0abdd7a2014-01-21 15:48:12 -0800480{
481 int overlap = 0, i;
482
483 for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--)
Dan Williams3b7a6412014-03-03 15:38:21 -0800484 if (radix_tree_tag_get(&dma_active_cacheline, cln, i))
Dan Williams0abdd7a2014-01-21 15:48:12 -0800485 overlap |= 1 << i;
486 return overlap;
487}
488
Dan Williams3b7a6412014-03-03 15:38:21 -0800489static int active_cacheline_set_overlap(phys_addr_t cln, int overlap)
Dan Williams0abdd7a2014-01-21 15:48:12 -0800490{
491 int i;
492
Dan Williams3b7a6412014-03-03 15:38:21 -0800493 if (overlap > ACTIVE_CACHELINE_MAX_OVERLAP || overlap < 0)
Dan Williams59f2e7d2014-01-29 14:05:53 -0800494 return overlap;
Dan Williams0abdd7a2014-01-21 15:48:12 -0800495
496 for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--)
497 if (overlap & 1 << i)
Dan Williams3b7a6412014-03-03 15:38:21 -0800498 radix_tree_tag_set(&dma_active_cacheline, cln, i);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800499 else
Dan Williams3b7a6412014-03-03 15:38:21 -0800500 radix_tree_tag_clear(&dma_active_cacheline, cln, i);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800501
502 return overlap;
503}
504
Dan Williams3b7a6412014-03-03 15:38:21 -0800505static void active_cacheline_inc_overlap(phys_addr_t cln)
Dan Williams0abdd7a2014-01-21 15:48:12 -0800506{
Dan Williams3b7a6412014-03-03 15:38:21 -0800507 int overlap = active_cacheline_read_overlap(cln);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800508
Dan Williams3b7a6412014-03-03 15:38:21 -0800509 overlap = active_cacheline_set_overlap(cln, ++overlap);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800510
511 /* If we overflowed the overlap counter then we're potentially
512 * leaking dma-mappings. Otherwise, if maps and unmaps are
513 * balanced then this overflow may cause false negatives in
Dan Williams3b7a6412014-03-03 15:38:21 -0800514 * debug_dma_assert_idle() as the cacheline may be marked idle
Dan Williams0abdd7a2014-01-21 15:48:12 -0800515 * prematurely.
516 */
Dan Williams3b7a6412014-03-03 15:38:21 -0800517 WARN_ONCE(overlap > ACTIVE_CACHELINE_MAX_OVERLAP,
518 "DMA-API: exceeded %d overlapping mappings of cacheline %pa\n",
519 ACTIVE_CACHELINE_MAX_OVERLAP, &cln);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800520}
521
Dan Williams3b7a6412014-03-03 15:38:21 -0800522static int active_cacheline_dec_overlap(phys_addr_t cln)
Dan Williams0abdd7a2014-01-21 15:48:12 -0800523{
Dan Williams3b7a6412014-03-03 15:38:21 -0800524 int overlap = active_cacheline_read_overlap(cln);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800525
Dan Williams3b7a6412014-03-03 15:38:21 -0800526 return active_cacheline_set_overlap(cln, --overlap);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800527}
528
Dan Williams3b7a6412014-03-03 15:38:21 -0800529static int active_cacheline_insert(struct dma_debug_entry *entry)
Dan Williams0abdd7a2014-01-21 15:48:12 -0800530{
Dan Williams3b7a6412014-03-03 15:38:21 -0800531 phys_addr_t cln = to_cacheline_number(entry);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800532 unsigned long flags;
533 int rc;
534
Dan Williams3b7a6412014-03-03 15:38:21 -0800535 /* If the device is not writing memory then we don't have any
536 * concerns about the cpu consuming stale data. This mitigates
537 * legitimate usages of overlapping mappings.
538 */
539 if (entry->direction == DMA_TO_DEVICE)
540 return 0;
541
Dan Williams0abdd7a2014-01-21 15:48:12 -0800542 spin_lock_irqsave(&radix_lock, flags);
Dan Williams3b7a6412014-03-03 15:38:21 -0800543 rc = radix_tree_insert(&dma_active_cacheline, cln, entry);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800544 if (rc == -EEXIST)
Dan Williams3b7a6412014-03-03 15:38:21 -0800545 active_cacheline_inc_overlap(cln);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800546 spin_unlock_irqrestore(&radix_lock, flags);
547
548 return rc;
549}
550
Dan Williams3b7a6412014-03-03 15:38:21 -0800551static void active_cacheline_remove(struct dma_debug_entry *entry)
Dan Williams0abdd7a2014-01-21 15:48:12 -0800552{
Dan Williams3b7a6412014-03-03 15:38:21 -0800553 phys_addr_t cln = to_cacheline_number(entry);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800554 unsigned long flags;
555
Dan Williams3b7a6412014-03-03 15:38:21 -0800556 /* ...mirror the insert case */
557 if (entry->direction == DMA_TO_DEVICE)
558 return;
559
Dan Williams0abdd7a2014-01-21 15:48:12 -0800560 spin_lock_irqsave(&radix_lock, flags);
Dan Williams59f2e7d2014-01-29 14:05:53 -0800561 /* since we are counting overlaps the final put of the
Dan Williams3b7a6412014-03-03 15:38:21 -0800562 * cacheline will occur when the overlap count is 0.
563 * active_cacheline_dec_overlap() returns -1 in that case
Dan Williams59f2e7d2014-01-29 14:05:53 -0800564 */
Dan Williams3b7a6412014-03-03 15:38:21 -0800565 if (active_cacheline_dec_overlap(cln) < 0)
566 radix_tree_delete(&dma_active_cacheline, cln);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800567 spin_unlock_irqrestore(&radix_lock, flags);
568}
569
570/**
571 * debug_dma_assert_idle() - assert that a page is not undergoing dma
Dan Williams3b7a6412014-03-03 15:38:21 -0800572 * @page: page to lookup in the dma_active_cacheline tree
Dan Williams0abdd7a2014-01-21 15:48:12 -0800573 *
574 * Place a call to this routine in cases where the cpu touching the page
575 * before the dma completes (page is dma_unmapped) will lead to data
576 * corruption.
577 */
578void debug_dma_assert_idle(struct page *page)
579{
Dan Williams3b7a6412014-03-03 15:38:21 -0800580 static struct dma_debug_entry *ents[CACHELINES_PER_PAGE];
581 struct dma_debug_entry *entry = NULL;
582 void **results = (void **) &ents;
583 unsigned int nents, i;
Dan Williams0abdd7a2014-01-21 15:48:12 -0800584 unsigned long flags;
Dan Williams3b7a6412014-03-03 15:38:21 -0800585 phys_addr_t cln;
Dan Williams0abdd7a2014-01-21 15:48:12 -0800586
Haggai Eranc9d120b2015-07-17 16:24:06 -0700587 if (dma_debug_disabled())
588 return;
589
Dan Williams0abdd7a2014-01-21 15:48:12 -0800590 if (!page)
591 return;
592
Dan Williams3b7a6412014-03-03 15:38:21 -0800593 cln = (phys_addr_t) page_to_pfn(page) << CACHELINE_PER_PAGE_SHIFT;
Dan Williams0abdd7a2014-01-21 15:48:12 -0800594 spin_lock_irqsave(&radix_lock, flags);
Dan Williams3b7a6412014-03-03 15:38:21 -0800595 nents = radix_tree_gang_lookup(&dma_active_cacheline, results, cln,
596 CACHELINES_PER_PAGE);
597 for (i = 0; i < nents; i++) {
598 phys_addr_t ent_cln = to_cacheline_number(ents[i]);
599
600 if (ent_cln == cln) {
601 entry = ents[i];
602 break;
603 } else if (ent_cln >= cln + CACHELINES_PER_PAGE)
604 break;
605 }
Dan Williams0abdd7a2014-01-21 15:48:12 -0800606 spin_unlock_irqrestore(&radix_lock, flags);
607
608 if (!entry)
609 return;
610
Dan Williams3b7a6412014-03-03 15:38:21 -0800611 cln = to_cacheline_number(entry);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800612 err_printk(entry->dev, entry,
Dan Williams3b7a6412014-03-03 15:38:21 -0800613 "DMA-API: cpu touching an active dma mapped cacheline [cln=%pa]\n",
614 &cln);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800615}
616
617/*
Joerg Roedel30dfa902009-01-09 12:34:49 +0100618 * Wrapper function for adding an entry to the hash.
619 * This function takes care of locking itself.
620 */
621static void add_dma_entry(struct dma_debug_entry *entry)
622{
623 struct hash_bucket *bucket;
624 unsigned long flags;
Dan Williams0abdd7a2014-01-21 15:48:12 -0800625 int rc;
Joerg Roedel30dfa902009-01-09 12:34:49 +0100626
627 bucket = get_hash_bucket(entry, &flags);
628 hash_bucket_add(bucket, entry);
629 put_hash_bucket(bucket, &flags);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800630
Dan Williams3b7a6412014-03-03 15:38:21 -0800631 rc = active_cacheline_insert(entry);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800632 if (rc == -ENOMEM) {
Dan Williams3b7a6412014-03-03 15:38:21 -0800633 pr_err("DMA-API: cacheline tracking ENOMEM, dma-debug disabled\n");
Dan Williams0abdd7a2014-01-21 15:48:12 -0800634 global_disable = true;
635 }
636
637 /* TODO: report -EEXIST errors here as overlapping mappings are
638 * not supported by the DMA API
639 */
Joerg Roedel30dfa902009-01-09 12:34:49 +0100640}
641
FUJITA Tomonorie6a1a892009-04-15 18:22:41 +0900642static struct dma_debug_entry *__dma_entry_alloc(void)
643{
644 struct dma_debug_entry *entry;
645
646 entry = list_entry(free_entries.next, struct dma_debug_entry, list);
647 list_del(&entry->list);
648 memset(entry, 0, sizeof(*entry));
649
650 num_free_entries -= 1;
651 if (num_free_entries < min_free_entries)
652 min_free_entries = num_free_entries;
653
654 return entry;
655}
656
Joerg Roedel3b1e79e2009-01-09 12:42:46 +0100657/* struct dma_entry allocator
658 *
659 * The next two functions implement the allocator for
660 * struct dma_debug_entries.
661 */
662static struct dma_debug_entry *dma_entry_alloc(void)
663{
Jakub Kicinski29cdd4e2012-04-04 03:19:10 +0200664 struct dma_debug_entry *entry;
Joerg Roedel3b1e79e2009-01-09 12:42:46 +0100665 unsigned long flags;
666
667 spin_lock_irqsave(&free_entries_lock, flags);
668
669 if (list_empty(&free_entries)) {
Joerg Roedel3b1e79e2009-01-09 12:42:46 +0100670 global_disable = true;
Jakub Kicinski29cdd4e2012-04-04 03:19:10 +0200671 spin_unlock_irqrestore(&free_entries_lock, flags);
Ville Syrjälä3017cd62016-05-26 15:16:25 -0700672 pr_err("DMA-API: debugging out of memory - disabling\n");
Jakub Kicinski29cdd4e2012-04-04 03:19:10 +0200673 return NULL;
Joerg Roedel3b1e79e2009-01-09 12:42:46 +0100674 }
675
FUJITA Tomonorie6a1a892009-04-15 18:22:41 +0900676 entry = __dma_entry_alloc();
Joerg Roedel3b1e79e2009-01-09 12:42:46 +0100677
Jakub Kicinski29cdd4e2012-04-04 03:19:10 +0200678 spin_unlock_irqrestore(&free_entries_lock, flags);
679
David Woodhouse6c132d12009-01-19 16:52:39 +0100680#ifdef CONFIG_STACKTRACE
681 entry->stacktrace.max_entries = DMA_DEBUG_STACKTRACE_ENTRIES;
682 entry->stacktrace.entries = entry->st_entries;
683 entry->stacktrace.skip = 2;
684 save_stack_trace(&entry->stacktrace);
685#endif
Joerg Roedel3b1e79e2009-01-09 12:42:46 +0100686
Joerg Roedel3b1e79e2009-01-09 12:42:46 +0100687 return entry;
688}
689
690static void dma_entry_free(struct dma_debug_entry *entry)
691{
692 unsigned long flags;
693
Dan Williams3b7a6412014-03-03 15:38:21 -0800694 active_cacheline_remove(entry);
Dan Williams0abdd7a2014-01-21 15:48:12 -0800695
Joerg Roedel3b1e79e2009-01-09 12:42:46 +0100696 /*
697 * add to beginning of the list - this way the entries are
698 * more likely cache hot when they are reallocated.
699 */
700 spin_lock_irqsave(&free_entries_lock, flags);
701 list_add(&entry->list, &free_entries);
702 num_free_entries += 1;
703 spin_unlock_irqrestore(&free_entries_lock, flags);
704}
705
FUJITA Tomonorie6a1a892009-04-15 18:22:41 +0900706int dma_debug_resize_entries(u32 num_entries)
707{
708 int i, delta, ret = 0;
709 unsigned long flags;
710 struct dma_debug_entry *entry;
711 LIST_HEAD(tmp);
712
713 spin_lock_irqsave(&free_entries_lock, flags);
714
715 if (nr_total_entries < num_entries) {
716 delta = num_entries - nr_total_entries;
717
718 spin_unlock_irqrestore(&free_entries_lock, flags);
719
720 for (i = 0; i < delta; i++) {
721 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
722 if (!entry)
723 break;
724
725 list_add_tail(&entry->list, &tmp);
726 }
727
728 spin_lock_irqsave(&free_entries_lock, flags);
729
730 list_splice(&tmp, &free_entries);
731 nr_total_entries += i;
732 num_free_entries += i;
733 } else {
734 delta = nr_total_entries - num_entries;
735
736 for (i = 0; i < delta && !list_empty(&free_entries); i++) {
737 entry = __dma_entry_alloc();
738 kfree(entry);
739 }
740
741 nr_total_entries -= i;
742 }
743
744 if (nr_total_entries != num_entries)
745 ret = 1;
746
747 spin_unlock_irqrestore(&free_entries_lock, flags);
748
749 return ret;
750}
751EXPORT_SYMBOL(dma_debug_resize_entries);
752
Joerg Roedel6bf07872009-01-09 12:54:42 +0100753/*
754 * DMA-API debugging init code
755 *
756 * The init code does two things:
757 * 1. Initialize core data structures
758 * 2. Preallocate a given number of dma_debug_entry structs
759 */
760
761static int prealloc_memory(u32 num_entries)
762{
763 struct dma_debug_entry *entry, *next_entry;
764 int i;
765
766 for (i = 0; i < num_entries; ++i) {
767 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
768 if (!entry)
769 goto out_err;
770
771 list_add_tail(&entry->list, &free_entries);
772 }
773
774 num_free_entries = num_entries;
775 min_free_entries = num_entries;
776
Joerg Roedele7ed70e2009-06-08 15:39:24 +0200777 pr_info("DMA-API: preallocated %d debug entries\n", num_entries);
Joerg Roedel6bf07872009-01-09 12:54:42 +0100778
779 return 0;
780
781out_err:
782
783 list_for_each_entry_safe(entry, next_entry, &free_entries, list) {
784 list_del(&entry->list);
785 kfree(entry);
786 }
787
788 return -ENOMEM;
789}
790
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200791static ssize_t filter_read(struct file *file, char __user *user_buf,
792 size_t count, loff_t *ppos)
793{
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200794 char buf[NAME_MAX_LEN + 1];
Joerg Roedelc17e2cf2009-06-08 15:19:29 +0200795 unsigned long flags;
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200796 int len;
797
798 if (!current_driver_name[0])
799 return 0;
800
801 /*
802 * We can't copy to userspace directly because current_driver_name can
803 * only be read under the driver_name_lock with irqs disabled. So
804 * create a temporary copy first.
805 */
806 read_lock_irqsave(&driver_name_lock, flags);
807 len = scnprintf(buf, NAME_MAX_LEN + 1, "%s\n", current_driver_name);
808 read_unlock_irqrestore(&driver_name_lock, flags);
809
810 return simple_read_from_buffer(user_buf, count, ppos, buf, len);
811}
812
813static ssize_t filter_write(struct file *file, const char __user *userbuf,
814 size_t count, loff_t *ppos)
815{
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200816 char buf[NAME_MAX_LEN];
Joerg Roedelc17e2cf2009-06-08 15:19:29 +0200817 unsigned long flags;
818 size_t len;
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200819 int i;
820
821 /*
822 * We can't copy from userspace directly. Access to
823 * current_driver_name is protected with a write_lock with irqs
824 * disabled. Since copy_from_user can fault and may sleep we
825 * need to copy to temporary buffer first
826 */
Joerg Roedele7ed70e2009-06-08 15:39:24 +0200827 len = min(count, (size_t)(NAME_MAX_LEN - 1));
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200828 if (copy_from_user(buf, userbuf, len))
829 return -EFAULT;
830
831 buf[len] = 0;
832
833 write_lock_irqsave(&driver_name_lock, flags);
834
Joerg Roedel312325092009-06-08 15:07:08 +0200835 /*
836 * Now handle the string we got from userspace very carefully.
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200837 * The rules are:
838 * - only use the first token we got
839 * - token delimiter is everything looking like a space
840 * character (' ', '\n', '\t' ...)
841 *
842 */
843 if (!isalnum(buf[0])) {
844 /*
Joerg Roedel312325092009-06-08 15:07:08 +0200845 * If the first character userspace gave us is not
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200846 * alphanumerical then assume the filter should be
847 * switched off.
848 */
849 if (current_driver_name[0])
Joerg Roedele7ed70e2009-06-08 15:39:24 +0200850 pr_info("DMA-API: switching off dma-debug driver filter\n");
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200851 current_driver_name[0] = 0;
852 current_driver = NULL;
853 goto out_unlock;
854 }
855
856 /*
857 * Now parse out the first token and use it as the name for the
858 * driver to filter for.
859 */
Dan Carpenter39a37ce2010-04-06 19:45:12 +0300860 for (i = 0; i < NAME_MAX_LEN - 1; ++i) {
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200861 current_driver_name[i] = buf[i];
862 if (isspace(buf[i]) || buf[i] == ' ' || buf[i] == 0)
863 break;
864 }
865 current_driver_name[i] = 0;
866 current_driver = NULL;
867
Joerg Roedele7ed70e2009-06-08 15:39:24 +0200868 pr_info("DMA-API: enable driver filter for driver [%s]\n",
869 current_driver_name);
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200870
871out_unlock:
872 write_unlock_irqrestore(&driver_name_lock, flags);
873
874 return count;
875}
876
Thiago Farinaaeb583d2010-01-18 18:57:33 -0500877static const struct file_operations filter_fops = {
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200878 .read = filter_read,
879 .write = filter_write,
Arnd Bergmann6038f372010-08-15 18:52:59 +0200880 .llseek = default_llseek,
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200881};
882
Joerg Roedel788dcfa2009-01-09 13:13:27 +0100883static int dma_debug_fs_init(void)
884{
885 dma_debug_dent = debugfs_create_dir("dma-api", NULL);
886 if (!dma_debug_dent) {
Joerg Roedele7ed70e2009-06-08 15:39:24 +0200887 pr_err("DMA-API: can not create debugfs directory\n");
Joerg Roedel788dcfa2009-01-09 13:13:27 +0100888 return -ENOMEM;
889 }
890
891 global_disable_dent = debugfs_create_bool("disabled", 0444,
892 dma_debug_dent,
Dan Carpenter68ee6d22012-06-27 12:08:55 +0300893 &global_disable);
Joerg Roedel788dcfa2009-01-09 13:13:27 +0100894 if (!global_disable_dent)
895 goto out_err;
896
897 error_count_dent = debugfs_create_u32("error_count", 0444,
898 dma_debug_dent, &error_count);
899 if (!error_count_dent)
900 goto out_err;
901
902 show_all_errors_dent = debugfs_create_u32("all_errors", 0644,
903 dma_debug_dent,
904 &show_all_errors);
905 if (!show_all_errors_dent)
906 goto out_err;
907
908 show_num_errors_dent = debugfs_create_u32("num_errors", 0644,
909 dma_debug_dent,
910 &show_num_errors);
911 if (!show_num_errors_dent)
912 goto out_err;
913
914 num_free_entries_dent = debugfs_create_u32("num_free_entries", 0444,
915 dma_debug_dent,
916 &num_free_entries);
917 if (!num_free_entries_dent)
918 goto out_err;
919
920 min_free_entries_dent = debugfs_create_u32("min_free_entries", 0444,
921 dma_debug_dent,
922 &min_free_entries);
923 if (!min_free_entries_dent)
924 goto out_err;
925
Joerg Roedel8a6fc702009-05-22 21:23:13 +0200926 filter_dent = debugfs_create_file("driver_filter", 0644,
927 dma_debug_dent, NULL, &filter_fops);
928 if (!filter_dent)
929 goto out_err;
930
Joerg Roedel788dcfa2009-01-09 13:13:27 +0100931 return 0;
932
933out_err:
934 debugfs_remove_recursive(dma_debug_dent);
935
936 return -ENOMEM;
937}
938
Stanislaw Gruszkaba4b87ad2011-03-31 08:08:09 -0400939static int device_dma_allocations(struct device *dev, struct dma_debug_entry **out_entry)
Joerg Roedeled888ae2009-05-22 17:16:04 +0200940{
941 struct dma_debug_entry *entry;
942 unsigned long flags;
943 int count = 0, i;
944
945 for (i = 0; i < HASH_SIZE; ++i) {
Pankaj Gupta6a5cd602017-05-03 14:51:28 -0700946 spin_lock_irqsave(&dma_entry_hash[i].lock, flags);
Joerg Roedeled888ae2009-05-22 17:16:04 +0200947 list_for_each_entry(entry, &dma_entry_hash[i].list, list) {
Stanislaw Gruszkaba4b87ad2011-03-31 08:08:09 -0400948 if (entry->dev == dev) {
Joerg Roedeled888ae2009-05-22 17:16:04 +0200949 count += 1;
Stanislaw Gruszkaba4b87ad2011-03-31 08:08:09 -0400950 *out_entry = entry;
951 }
Joerg Roedeled888ae2009-05-22 17:16:04 +0200952 }
Pankaj Gupta6a5cd602017-05-03 14:51:28 -0700953 spin_unlock_irqrestore(&dma_entry_hash[i].lock, flags);
Joerg Roedeled888ae2009-05-22 17:16:04 +0200954 }
955
956 return count;
957}
958
Ingo Molnara8fe9ea2009-12-31 15:16:23 +0100959static int dma_debug_device_change(struct notifier_block *nb, unsigned long action, void *data)
Joerg Roedeled888ae2009-05-22 17:16:04 +0200960{
961 struct device *dev = data;
Stanislaw Gruszkaba4b87ad2011-03-31 08:08:09 -0400962 struct dma_debug_entry *uninitialized_var(entry);
Joerg Roedeled888ae2009-05-22 17:16:04 +0200963 int count;
964
Florian Fainelli01ce18b2014-12-10 15:41:23 -0800965 if (dma_debug_disabled())
Ingo Molnara8fe9ea2009-12-31 15:16:23 +0100966 return 0;
Joerg Roedeled888ae2009-05-22 17:16:04 +0200967
968 switch (action) {
969 case BUS_NOTIFY_UNBOUND_DRIVER:
Stanislaw Gruszkaba4b87ad2011-03-31 08:08:09 -0400970 count = device_dma_allocations(dev, &entry);
Joerg Roedeled888ae2009-05-22 17:16:04 +0200971 if (count == 0)
972 break;
Stanislaw Gruszkaba4b87ad2011-03-31 08:08:09 -0400973 err_printk(dev, entry, "DMA-API: device driver has pending "
Joerg Roedeled888ae2009-05-22 17:16:04 +0200974 "DMA allocations while released from device "
Stanislaw Gruszkaba4b87ad2011-03-31 08:08:09 -0400975 "[count=%d]\n"
976 "One of leaked entries details: "
977 "[device address=0x%016llx] [size=%llu bytes] "
978 "[mapped with %s] [mapped as %s]\n",
979 count, entry->dev_addr, entry->size,
980 dir2name[entry->direction], type2name[entry->type]);
Joerg Roedeled888ae2009-05-22 17:16:04 +0200981 break;
982 default:
983 break;
984 }
985
986 return 0;
987}
988
Joerg Roedel41531c82009-03-16 17:32:14 +0100989void dma_debug_add_bus(struct bus_type *bus)
990{
Joerg Roedeled888ae2009-05-22 17:16:04 +0200991 struct notifier_block *nb;
992
Florian Fainelli01ce18b2014-12-10 15:41:23 -0800993 if (dma_debug_disabled())
Shaun Ruffellf797d982009-12-17 18:00:36 -0600994 return;
995
Joerg Roedeled888ae2009-05-22 17:16:04 +0200996 nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL);
997 if (nb == NULL) {
Joerg Roedele7ed70e2009-06-08 15:39:24 +0200998 pr_err("dma_debug_add_bus: out of memory\n");
Joerg Roedeled888ae2009-05-22 17:16:04 +0200999 return;
1000 }
1001
1002 nb->notifier_call = dma_debug_device_change;
1003
1004 bus_register_notifier(bus, nb);
Joerg Roedel41531c82009-03-16 17:32:14 +01001005}
Joerg Roedel788dcfa2009-01-09 13:13:27 +01001006
Joerg Roedel6bf07872009-01-09 12:54:42 +01001007/*
1008 * Let the architectures decide how many entries should be preallocated.
1009 */
1010void dma_debug_init(u32 num_entries)
1011{
1012 int i;
1013
Florian Fainelli2ce8e7e2014-12-10 15:41:25 -08001014 /* Do not use dma_debug_initialized here, since we really want to be
1015 * called to set dma_debug_initialized
1016 */
1017 if (global_disable)
Joerg Roedel6bf07872009-01-09 12:54:42 +01001018 return;
1019
1020 for (i = 0; i < HASH_SIZE; ++i) {
1021 INIT_LIST_HEAD(&dma_entry_hash[i].list);
Ingo Molnarb0a5b832009-06-16 16:11:14 +02001022 spin_lock_init(&dma_entry_hash[i].lock);
Joerg Roedel6bf07872009-01-09 12:54:42 +01001023 }
1024
Joerg Roedel788dcfa2009-01-09 13:13:27 +01001025 if (dma_debug_fs_init() != 0) {
Joerg Roedele7ed70e2009-06-08 15:39:24 +02001026 pr_err("DMA-API: error creating debugfs entries - disabling\n");
Joerg Roedel788dcfa2009-01-09 13:13:27 +01001027 global_disable = true;
1028
1029 return;
1030 }
1031
Joerg Roedel59d3daa2009-01-09 13:01:56 +01001032 if (req_entries)
1033 num_entries = req_entries;
1034
Joerg Roedel6bf07872009-01-09 12:54:42 +01001035 if (prealloc_memory(num_entries) != 0) {
Joerg Roedele7ed70e2009-06-08 15:39:24 +02001036 pr_err("DMA-API: debugging out of memory error - disabled\n");
Joerg Roedel6bf07872009-01-09 12:54:42 +01001037 global_disable = true;
1038
1039 return;
1040 }
1041
FUJITA Tomonorie6a1a892009-04-15 18:22:41 +09001042 nr_total_entries = num_free_entries;
1043
Florian Fainelli2ce8e7e2014-12-10 15:41:25 -08001044 dma_debug_initialized = true;
1045
Joerg Roedele7ed70e2009-06-08 15:39:24 +02001046 pr_info("DMA-API: debugging enabled by kernel config\n");
Joerg Roedel6bf07872009-01-09 12:54:42 +01001047}
1048
Joerg Roedel59d3daa2009-01-09 13:01:56 +01001049static __init int dma_debug_cmdline(char *str)
1050{
1051 if (!str)
1052 return -EINVAL;
1053
1054 if (strncmp(str, "off", 3) == 0) {
Joerg Roedele7ed70e2009-06-08 15:39:24 +02001055 pr_info("DMA-API: debugging disabled on kernel command line\n");
Joerg Roedel59d3daa2009-01-09 13:01:56 +01001056 global_disable = true;
1057 }
1058
1059 return 0;
1060}
1061
1062static __init int dma_debug_entries_cmdline(char *str)
1063{
1064 int res;
1065
1066 if (!str)
1067 return -EINVAL;
1068
1069 res = get_option(&str, &req_entries);
1070
1071 if (!res)
1072 req_entries = 0;
1073
1074 return 0;
1075}
1076
1077__setup("dma_debug=", dma_debug_cmdline);
1078__setup("dma_debug_entries=", dma_debug_entries_cmdline);
1079
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001080static void check_unmap(struct dma_debug_entry *ref)
1081{
1082 struct dma_debug_entry *entry;
1083 struct hash_bucket *bucket;
1084 unsigned long flags;
1085
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001086 bucket = get_hash_bucket(ref, &flags);
Neil Hormanc6a21d02011-08-08 15:13:54 -04001087 entry = bucket_find_exact(bucket, ref);
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001088
1089 if (!entry) {
Alexander Duyck8d640a52013-03-22 15:04:48 -07001090 /* must drop lock before calling dma_mapping_error */
1091 put_hash_bucket(bucket, &flags);
1092
Shuah Khanbfe0fb02012-11-03 17:00:07 -06001093 if (dma_mapping_error(ref->dev, ref->dev_addr)) {
1094 err_printk(ref->dev, NULL,
Alexander Duyck8d640a52013-03-22 15:04:48 -07001095 "DMA-API: device driver tries to free an "
1096 "invalid DMA memory address\n");
1097 } else {
1098 err_printk(ref->dev, NULL,
1099 "DMA-API: device driver tries to free DMA "
1100 "memory it has not allocated [device "
1101 "address=0x%016llx] [size=%llu bytes]\n",
1102 ref->dev_addr, ref->size);
Shuah Khanbfe0fb02012-11-03 17:00:07 -06001103 }
Alexander Duyck8d640a52013-03-22 15:04:48 -07001104 return;
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001105 }
1106
1107 if (ref->size != entry->size) {
David Woodhouse6c132d12009-01-19 16:52:39 +01001108 err_printk(ref->dev, entry, "DMA-API: device driver frees "
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001109 "DMA memory with different size "
1110 "[device address=0x%016llx] [map size=%llu bytes] "
1111 "[unmap size=%llu bytes]\n",
1112 ref->dev_addr, entry->size, ref->size);
1113 }
1114
1115 if (ref->type != entry->type) {
David Woodhouse6c132d12009-01-19 16:52:39 +01001116 err_printk(ref->dev, entry, "DMA-API: device driver frees "
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001117 "DMA memory with wrong function "
1118 "[device address=0x%016llx] [size=%llu bytes] "
1119 "[mapped as %s] [unmapped as %s]\n",
1120 ref->dev_addr, ref->size,
1121 type2name[entry->type], type2name[ref->type]);
1122 } else if ((entry->type == dma_debug_coherent) &&
Dan Williams0abdd7a2014-01-21 15:48:12 -08001123 (phys_addr(ref) != phys_addr(entry))) {
David Woodhouse6c132d12009-01-19 16:52:39 +01001124 err_printk(ref->dev, entry, "DMA-API: device driver frees "
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001125 "DMA memory with different CPU address "
1126 "[device address=0x%016llx] [size=%llu bytes] "
Joerg Roedel59a40e702009-10-29 16:25:50 +01001127 "[cpu alloc address=0x%016llx] "
1128 "[cpu free address=0x%016llx]",
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001129 ref->dev_addr, ref->size,
Dan Williams0abdd7a2014-01-21 15:48:12 -08001130 phys_addr(entry),
1131 phys_addr(ref));
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001132 }
1133
1134 if (ref->sg_call_ents && ref->type == dma_debug_sg &&
1135 ref->sg_call_ents != entry->sg_call_ents) {
David Woodhouse6c132d12009-01-19 16:52:39 +01001136 err_printk(ref->dev, entry, "DMA-API: device driver frees "
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001137 "DMA sg list with different entry count "
1138 "[map count=%d] [unmap count=%d]\n",
1139 entry->sg_call_ents, ref->sg_call_ents);
1140 }
1141
1142 /*
1143 * This may be no bug in reality - but most implementations of the
1144 * DMA API don't handle this properly, so check for it here
1145 */
1146 if (ref->direction != entry->direction) {
David Woodhouse6c132d12009-01-19 16:52:39 +01001147 err_printk(ref->dev, entry, "DMA-API: device driver frees "
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001148 "DMA memory with different direction "
1149 "[device address=0x%016llx] [size=%llu bytes] "
1150 "[mapped with %s] [unmapped with %s]\n",
1151 ref->dev_addr, ref->size,
1152 dir2name[entry->direction],
1153 dir2name[ref->direction]);
1154 }
1155
Miles Chena5759b22017-02-22 15:40:09 -08001156 /*
1157 * Drivers should use dma_mapping_error() to check the returned
1158 * addresses of dma_map_single() and dma_map_page().
1159 * If not, print this warning message. See Documentation/DMA-API.txt.
1160 */
Shuah Khan6c9c6d62012-10-08 11:08:06 -06001161 if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {
1162 err_printk(ref->dev, entry,
1163 "DMA-API: device driver failed to check map error"
1164 "[device address=0x%016llx] [size=%llu bytes] "
1165 "[mapped as %s]",
1166 ref->dev_addr, ref->size,
1167 type2name[entry->type]);
1168 }
1169
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001170 hash_bucket_del(entry);
1171 dma_entry_free(entry);
1172
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001173 put_hash_bucket(bucket, &flags);
1174}
1175
Andy Lutomirskib4a0f532016-08-11 02:35:22 -07001176static void check_for_stack(struct device *dev,
1177 struct page *page, size_t offset)
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001178{
Andy Lutomirskib4a0f532016-08-11 02:35:22 -07001179 void *addr;
1180 struct vm_struct *stack_vm_area = task_stack_vm_area(current);
1181
1182 if (!stack_vm_area) {
1183 /* Stack is direct-mapped. */
1184 if (PageHighMem(page))
1185 return;
1186 addr = page_address(page) + offset;
1187 if (object_is_on_stack(addr))
1188 err_printk(dev, NULL, "DMA-API: device driver maps memory from stack [addr=%p]\n", addr);
1189 } else {
1190 /* Stack is vmalloced. */
1191 int i;
1192
1193 for (i = 0; i < stack_vm_area->nr_pages; i++) {
1194 if (page != stack_vm_area->pages[i])
1195 continue;
1196
1197 addr = (u8 *)current->stack + i * PAGE_SIZE + offset;
1198 err_printk(dev, NULL, "DMA-API: device driver maps memory from stack [probable addr=%p]\n", addr);
1199 break;
1200 }
1201 }
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001202}
1203
Ingo Molnarf39d1b92009-07-10 21:38:02 +02001204static inline bool overlap(void *addr, unsigned long len, void *start, void *end)
Joerg Roedel2e34bde2009-03-16 16:51:55 +01001205{
Ingo Molnarf39d1b92009-07-10 21:38:02 +02001206 unsigned long a1 = (unsigned long)addr;
1207 unsigned long b1 = a1 + len;
1208 unsigned long a2 = (unsigned long)start;
1209 unsigned long b2 = (unsigned long)end;
Joerg Roedel2e34bde2009-03-16 16:51:55 +01001210
Ingo Molnarf39d1b92009-07-10 21:38:02 +02001211 return !(b1 <= a2 || a1 >= b2);
Joerg Roedel2e34bde2009-03-16 16:51:55 +01001212}
1213
Ingo Molnarf39d1b92009-07-10 21:38:02 +02001214static void check_for_illegal_area(struct device *dev, void *addr, unsigned long len)
Joerg Roedel2e34bde2009-03-16 16:51:55 +01001215{
Laura Abbottea535e42016-01-14 15:16:50 -08001216 if (overlap(addr, len, _stext, _etext) ||
Ingo Molnarf39d1b92009-07-10 21:38:02 +02001217 overlap(addr, len, __start_rodata, __end_rodata))
1218 err_printk(dev, NULL, "DMA-API: device driver maps memory from kernel text or rodata [addr=%p] [len=%lu]\n", addr, len);
Joerg Roedel2e34bde2009-03-16 16:51:55 +01001219}
1220
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001221static void check_sync(struct device *dev,
1222 struct dma_debug_entry *ref,
1223 bool to_cpu)
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001224{
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001225 struct dma_debug_entry *entry;
1226 struct hash_bucket *bucket;
1227 unsigned long flags;
1228
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001229 bucket = get_hash_bucket(ref, &flags);
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001230
Neil Hormanc6a21d02011-08-08 15:13:54 -04001231 entry = bucket_find_contain(&bucket, ref, &flags);
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001232
1233 if (!entry) {
David Woodhouse6c132d12009-01-19 16:52:39 +01001234 err_printk(dev, NULL, "DMA-API: device driver tries "
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001235 "to sync DMA memory it has not allocated "
1236 "[device address=0x%016llx] [size=%llu bytes]\n",
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001237 (unsigned long long)ref->dev_addr, ref->size);
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001238 goto out;
1239 }
1240
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001241 if (ref->size > entry->size) {
David Woodhouse6c132d12009-01-19 16:52:39 +01001242 err_printk(dev, entry, "DMA-API: device driver syncs"
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001243 " DMA memory outside allocated range "
1244 "[device address=0x%016llx] "
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001245 "[allocation size=%llu bytes] "
1246 "[sync offset+size=%llu]\n",
1247 entry->dev_addr, entry->size,
1248 ref->size);
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001249 }
1250
Krzysztof Halasa42d53b42010-01-08 14:42:36 -08001251 if (entry->direction == DMA_BIDIRECTIONAL)
1252 goto out;
1253
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001254 if (ref->direction != entry->direction) {
David Woodhouse6c132d12009-01-19 16:52:39 +01001255 err_printk(dev, entry, "DMA-API: device driver syncs "
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001256 "DMA memory with different direction "
1257 "[device address=0x%016llx] [size=%llu bytes] "
1258 "[mapped with %s] [synced with %s]\n",
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001259 (unsigned long long)ref->dev_addr, entry->size,
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001260 dir2name[entry->direction],
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001261 dir2name[ref->direction]);
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001262 }
1263
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001264 if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) &&
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001265 !(ref->direction == DMA_TO_DEVICE))
David Woodhouse6c132d12009-01-19 16:52:39 +01001266 err_printk(dev, entry, "DMA-API: device driver syncs "
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001267 "device read-only DMA memory for cpu "
1268 "[device address=0x%016llx] [size=%llu bytes] "
1269 "[mapped with %s] [synced with %s]\n",
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001270 (unsigned long long)ref->dev_addr, entry->size,
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001271 dir2name[entry->direction],
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001272 dir2name[ref->direction]);
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001273
1274 if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) &&
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001275 !(ref->direction == DMA_FROM_DEVICE))
David Woodhouse6c132d12009-01-19 16:52:39 +01001276 err_printk(dev, entry, "DMA-API: device driver syncs "
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001277 "device write-only DMA memory to device "
1278 "[device address=0x%016llx] [size=%llu bytes] "
1279 "[mapped with %s] [synced with %s]\n",
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001280 (unsigned long long)ref->dev_addr, entry->size,
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001281 dir2name[entry->direction],
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001282 dir2name[ref->direction]);
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001283
Robin Murphy7f830642015-11-06 16:32:55 -08001284 if (ref->sg_call_ents && ref->type == dma_debug_sg &&
1285 ref->sg_call_ents != entry->sg_call_ents) {
1286 err_printk(ref->dev, entry, "DMA-API: device driver syncs "
1287 "DMA sg list with different entry count "
1288 "[map count=%d] [sync count=%d]\n",
1289 entry->sg_call_ents, ref->sg_call_ents);
1290 }
1291
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001292out:
1293 put_hash_bucket(bucket, &flags);
Joerg Roedel2d62ece2009-01-09 14:10:26 +01001294}
1295
Joerg Roedelf62bc982009-01-09 14:14:49 +01001296void debug_dma_map_page(struct device *dev, struct page *page, size_t offset,
1297 size_t size, int direction, dma_addr_t dma_addr,
1298 bool map_single)
1299{
1300 struct dma_debug_entry *entry;
1301
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001302 if (unlikely(dma_debug_disabled()))
Joerg Roedelf62bc982009-01-09 14:14:49 +01001303 return;
1304
Shuah Khanbfe0fb02012-11-03 17:00:07 -06001305 if (dma_mapping_error(dev, dma_addr))
Joerg Roedelf62bc982009-01-09 14:14:49 +01001306 return;
1307
1308 entry = dma_entry_alloc();
1309 if (!entry)
1310 return;
1311
1312 entry->dev = dev;
1313 entry->type = dma_debug_page;
Dan Williams0abdd7a2014-01-21 15:48:12 -08001314 entry->pfn = page_to_pfn(page);
1315 entry->offset = offset,
Joerg Roedelf62bc982009-01-09 14:14:49 +01001316 entry->dev_addr = dma_addr;
1317 entry->size = size;
1318 entry->direction = direction;
Shuah Khan6c9c6d62012-10-08 11:08:06 -06001319 entry->map_err_type = MAP_ERR_NOT_CHECKED;
Joerg Roedelf62bc982009-01-09 14:14:49 +01001320
Joerg Roedel9537a482009-03-23 15:35:08 +01001321 if (map_single)
Joerg Roedelf62bc982009-01-09 14:14:49 +01001322 entry->type = dma_debug_single;
Joerg Roedel9537a482009-03-23 15:35:08 +01001323
Andy Lutomirskib4a0f532016-08-11 02:35:22 -07001324 check_for_stack(dev, page, offset);
1325
Joerg Roedel9537a482009-03-23 15:35:08 +01001326 if (!PageHighMem(page)) {
Ingo Molnarf39d1b92009-07-10 21:38:02 +02001327 void *addr = page_address(page) + offset;
1328
Joerg Roedel2e34bde2009-03-16 16:51:55 +01001329 check_for_illegal_area(dev, addr, size);
Joerg Roedelf62bc982009-01-09 14:14:49 +01001330 }
1331
1332 add_dma_entry(entry);
1333}
1334EXPORT_SYMBOL(debug_dma_map_page);
1335
Shuah Khan6c9c6d62012-10-08 11:08:06 -06001336void debug_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
1337{
1338 struct dma_debug_entry ref;
1339 struct dma_debug_entry *entry;
1340 struct hash_bucket *bucket;
1341 unsigned long flags;
1342
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001343 if (unlikely(dma_debug_disabled()))
Shuah Khan6c9c6d62012-10-08 11:08:06 -06001344 return;
1345
1346 ref.dev = dev;
1347 ref.dev_addr = dma_addr;
1348 bucket = get_hash_bucket(&ref, &flags);
Shuah Khan6c9c6d62012-10-08 11:08:06 -06001349
Alexander Duyck96e7d7a2013-03-22 15:04:49 -07001350 list_for_each_entry(entry, &bucket->list, list) {
1351 if (!exact_match(&ref, entry))
1352 continue;
Shuah Khan6c9c6d62012-10-08 11:08:06 -06001353
Alexander Duyck96e7d7a2013-03-22 15:04:49 -07001354 /*
1355 * The same physical address can be mapped multiple
1356 * times. Without a hardware IOMMU this results in the
1357 * same device addresses being put into the dma-debug
1358 * hash multiple times too. This can result in false
1359 * positives being reported. Therefore we implement a
1360 * best-fit algorithm here which updates the first entry
1361 * from the hash which fits the reference value and is
1362 * not currently listed as being checked.
1363 */
1364 if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {
1365 entry->map_err_type = MAP_ERR_CHECKED;
1366 break;
1367 }
1368 }
1369
Shuah Khan6c9c6d62012-10-08 11:08:06 -06001370 put_hash_bucket(bucket, &flags);
1371}
1372EXPORT_SYMBOL(debug_dma_mapping_error);
1373
Joerg Roedelf62bc982009-01-09 14:14:49 +01001374void debug_dma_unmap_page(struct device *dev, dma_addr_t addr,
1375 size_t size, int direction, bool map_single)
1376{
1377 struct dma_debug_entry ref = {
1378 .type = dma_debug_page,
1379 .dev = dev,
1380 .dev_addr = addr,
1381 .size = size,
1382 .direction = direction,
1383 };
1384
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001385 if (unlikely(dma_debug_disabled()))
Joerg Roedelf62bc982009-01-09 14:14:49 +01001386 return;
1387
1388 if (map_single)
1389 ref.type = dma_debug_single;
1390
1391 check_unmap(&ref);
1392}
1393EXPORT_SYMBOL(debug_dma_unmap_page);
1394
Joerg Roedel972aa452009-01-09 14:19:54 +01001395void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
1396 int nents, int mapped_ents, int direction)
1397{
1398 struct dma_debug_entry *entry;
1399 struct scatterlist *s;
1400 int i;
1401
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001402 if (unlikely(dma_debug_disabled()))
Joerg Roedel972aa452009-01-09 14:19:54 +01001403 return;
1404
1405 for_each_sg(sg, s, mapped_ents, i) {
1406 entry = dma_entry_alloc();
1407 if (!entry)
1408 return;
1409
1410 entry->type = dma_debug_sg;
1411 entry->dev = dev;
Dan Williams0abdd7a2014-01-21 15:48:12 -08001412 entry->pfn = page_to_pfn(sg_page(s));
1413 entry->offset = s->offset,
FUJITA Tomonori884d0592009-05-27 09:43:02 +09001414 entry->size = sg_dma_len(s);
FUJITA Tomonori15aedea2009-05-27 09:43:01 +09001415 entry->dev_addr = sg_dma_address(s);
Joerg Roedel972aa452009-01-09 14:19:54 +01001416 entry->direction = direction;
1417 entry->sg_call_ents = nents;
1418 entry->sg_mapped_ents = mapped_ents;
1419
Andy Lutomirskib4a0f532016-08-11 02:35:22 -07001420 check_for_stack(dev, sg_page(s), s->offset);
1421
Joerg Roedel9537a482009-03-23 15:35:08 +01001422 if (!PageHighMem(sg_page(s))) {
FUJITA Tomonori884d0592009-05-27 09:43:02 +09001423 check_for_illegal_area(dev, sg_virt(s), sg_dma_len(s));
Joerg Roedel9537a482009-03-23 15:35:08 +01001424 }
Joerg Roedel972aa452009-01-09 14:19:54 +01001425
1426 add_dma_entry(entry);
1427 }
1428}
1429EXPORT_SYMBOL(debug_dma_map_sg);
1430
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001431static int get_nr_mapped_entries(struct device *dev,
1432 struct dma_debug_entry *ref)
FUJITA Tomonori88f39072009-05-27 09:43:03 +09001433{
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001434 struct dma_debug_entry *entry;
FUJITA Tomonori88f39072009-05-27 09:43:03 +09001435 struct hash_bucket *bucket;
1436 unsigned long flags;
Joerg Roedelc17e2cf2009-06-08 15:19:29 +02001437 int mapped_ents;
FUJITA Tomonori88f39072009-05-27 09:43:03 +09001438
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001439 bucket = get_hash_bucket(ref, &flags);
Neil Hormanc6a21d02011-08-08 15:13:54 -04001440 entry = bucket_find_exact(bucket, ref);
Joerg Roedelc17e2cf2009-06-08 15:19:29 +02001441 mapped_ents = 0;
1442
FUJITA Tomonori88f39072009-05-27 09:43:03 +09001443 if (entry)
1444 mapped_ents = entry->sg_mapped_ents;
1445 put_hash_bucket(bucket, &flags);
1446
1447 return mapped_ents;
1448}
1449
Joerg Roedel972aa452009-01-09 14:19:54 +01001450void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
1451 int nelems, int dir)
1452{
Joerg Roedel972aa452009-01-09 14:19:54 +01001453 struct scatterlist *s;
1454 int mapped_ents = 0, i;
Joerg Roedel972aa452009-01-09 14:19:54 +01001455
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001456 if (unlikely(dma_debug_disabled()))
Joerg Roedel972aa452009-01-09 14:19:54 +01001457 return;
1458
1459 for_each_sg(sglist, s, nelems, i) {
1460
1461 struct dma_debug_entry ref = {
1462 .type = dma_debug_sg,
1463 .dev = dev,
Dan Williams0abdd7a2014-01-21 15:48:12 -08001464 .pfn = page_to_pfn(sg_page(s)),
1465 .offset = s->offset,
FUJITA Tomonori15aedea2009-05-27 09:43:01 +09001466 .dev_addr = sg_dma_address(s),
FUJITA Tomonori884d0592009-05-27 09:43:02 +09001467 .size = sg_dma_len(s),
Joerg Roedel972aa452009-01-09 14:19:54 +01001468 .direction = dir,
Joerg Roedele5e8c5b2009-06-11 10:03:42 +02001469 .sg_call_ents = nelems,
Joerg Roedel972aa452009-01-09 14:19:54 +01001470 };
1471
1472 if (mapped_ents && i >= mapped_ents)
1473 break;
1474
Joerg Roedele5e8c5b2009-06-11 10:03:42 +02001475 if (!i)
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001476 mapped_ents = get_nr_mapped_entries(dev, &ref);
Joerg Roedel972aa452009-01-09 14:19:54 +01001477
1478 check_unmap(&ref);
1479 }
1480}
1481EXPORT_SYMBOL(debug_dma_unmap_sg);
1482
Joerg Roedel6bfd4492009-01-09 14:38:50 +01001483void debug_dma_alloc_coherent(struct device *dev, size_t size,
1484 dma_addr_t dma_addr, void *virt)
1485{
1486 struct dma_debug_entry *entry;
1487
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001488 if (unlikely(dma_debug_disabled()))
Joerg Roedel6bfd4492009-01-09 14:38:50 +01001489 return;
1490
1491 if (unlikely(virt == NULL))
1492 return;
1493
1494 entry = dma_entry_alloc();
1495 if (!entry)
1496 return;
1497
1498 entry->type = dma_debug_coherent;
1499 entry->dev = dev;
Dan Williams0abdd7a2014-01-21 15:48:12 -08001500 entry->pfn = page_to_pfn(virt_to_page(virt));
Geliang Tange57d0552017-04-22 09:18:05 +08001501 entry->offset = offset_in_page(virt);
Joerg Roedel6bfd4492009-01-09 14:38:50 +01001502 entry->size = size;
1503 entry->dev_addr = dma_addr;
1504 entry->direction = DMA_BIDIRECTIONAL;
1505
1506 add_dma_entry(entry);
1507}
1508EXPORT_SYMBOL(debug_dma_alloc_coherent);
1509
1510void debug_dma_free_coherent(struct device *dev, size_t size,
1511 void *virt, dma_addr_t addr)
1512{
1513 struct dma_debug_entry ref = {
1514 .type = dma_debug_coherent,
1515 .dev = dev,
Dan Williams0abdd7a2014-01-21 15:48:12 -08001516 .pfn = page_to_pfn(virt_to_page(virt)),
Geliang Tange57d0552017-04-22 09:18:05 +08001517 .offset = offset_in_page(virt),
Joerg Roedel6bfd4492009-01-09 14:38:50 +01001518 .dev_addr = addr,
1519 .size = size,
1520 .direction = DMA_BIDIRECTIONAL,
1521 };
1522
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001523 if (unlikely(dma_debug_disabled()))
Joerg Roedel6bfd4492009-01-09 14:38:50 +01001524 return;
1525
1526 check_unmap(&ref);
1527}
1528EXPORT_SYMBOL(debug_dma_free_coherent);
1529
Niklas Söderlund0e74b342016-08-10 13:22:15 +02001530void debug_dma_map_resource(struct device *dev, phys_addr_t addr, size_t size,
1531 int direction, dma_addr_t dma_addr)
1532{
1533 struct dma_debug_entry *entry;
1534
1535 if (unlikely(dma_debug_disabled()))
1536 return;
1537
1538 entry = dma_entry_alloc();
1539 if (!entry)
1540 return;
1541
1542 entry->type = dma_debug_resource;
1543 entry->dev = dev;
Niklas Söderlund2e0cc302016-09-29 21:59:15 +02001544 entry->pfn = PHYS_PFN(addr);
Niklas Söderlund0e74b342016-08-10 13:22:15 +02001545 entry->offset = offset_in_page(addr);
1546 entry->size = size;
1547 entry->dev_addr = dma_addr;
1548 entry->direction = direction;
1549 entry->map_err_type = MAP_ERR_NOT_CHECKED;
1550
1551 add_dma_entry(entry);
1552}
1553EXPORT_SYMBOL(debug_dma_map_resource);
1554
1555void debug_dma_unmap_resource(struct device *dev, dma_addr_t dma_addr,
1556 size_t size, int direction)
1557{
1558 struct dma_debug_entry ref = {
1559 .type = dma_debug_resource,
1560 .dev = dev,
1561 .dev_addr = dma_addr,
1562 .size = size,
1563 .direction = direction,
1564 };
1565
1566 if (unlikely(dma_debug_disabled()))
1567 return;
1568
1569 check_unmap(&ref);
1570}
1571EXPORT_SYMBOL(debug_dma_unmap_resource);
1572
Joerg Roedelb9d23172009-01-09 14:43:04 +01001573void debug_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
1574 size_t size, int direction)
1575{
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001576 struct dma_debug_entry ref;
1577
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001578 if (unlikely(dma_debug_disabled()))
Joerg Roedelb9d23172009-01-09 14:43:04 +01001579 return;
1580
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001581 ref.type = dma_debug_single;
1582 ref.dev = dev;
1583 ref.dev_addr = dma_handle;
1584 ref.size = size;
1585 ref.direction = direction;
1586 ref.sg_call_ents = 0;
1587
1588 check_sync(dev, &ref, true);
Joerg Roedelb9d23172009-01-09 14:43:04 +01001589}
1590EXPORT_SYMBOL(debug_dma_sync_single_for_cpu);
1591
1592void debug_dma_sync_single_for_device(struct device *dev,
1593 dma_addr_t dma_handle, size_t size,
1594 int direction)
1595{
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001596 struct dma_debug_entry ref;
1597
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001598 if (unlikely(dma_debug_disabled()))
Joerg Roedelb9d23172009-01-09 14:43:04 +01001599 return;
1600
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001601 ref.type = dma_debug_single;
1602 ref.dev = dev;
1603 ref.dev_addr = dma_handle;
1604 ref.size = size;
1605 ref.direction = direction;
1606 ref.sg_call_ents = 0;
1607
1608 check_sync(dev, &ref, false);
Joerg Roedelb9d23172009-01-09 14:43:04 +01001609}
1610EXPORT_SYMBOL(debug_dma_sync_single_for_device);
1611
Joerg Roedel948408b2009-01-09 14:55:38 +01001612void debug_dma_sync_single_range_for_cpu(struct device *dev,
1613 dma_addr_t dma_handle,
1614 unsigned long offset, size_t size,
1615 int direction)
1616{
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001617 struct dma_debug_entry ref;
1618
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001619 if (unlikely(dma_debug_disabled()))
Joerg Roedel948408b2009-01-09 14:55:38 +01001620 return;
1621
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001622 ref.type = dma_debug_single;
1623 ref.dev = dev;
1624 ref.dev_addr = dma_handle;
1625 ref.size = offset + size;
1626 ref.direction = direction;
1627 ref.sg_call_ents = 0;
1628
1629 check_sync(dev, &ref, true);
Joerg Roedel948408b2009-01-09 14:55:38 +01001630}
1631EXPORT_SYMBOL(debug_dma_sync_single_range_for_cpu);
1632
1633void debug_dma_sync_single_range_for_device(struct device *dev,
1634 dma_addr_t dma_handle,
1635 unsigned long offset,
1636 size_t size, int direction)
1637{
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001638 struct dma_debug_entry ref;
1639
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001640 if (unlikely(dma_debug_disabled()))
Joerg Roedel948408b2009-01-09 14:55:38 +01001641 return;
1642
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001643 ref.type = dma_debug_single;
1644 ref.dev = dev;
1645 ref.dev_addr = dma_handle;
1646 ref.size = offset + size;
1647 ref.direction = direction;
1648 ref.sg_call_ents = 0;
1649
1650 check_sync(dev, &ref, false);
Joerg Roedel948408b2009-01-09 14:55:38 +01001651}
1652EXPORT_SYMBOL(debug_dma_sync_single_range_for_device);
1653
Joerg Roedela31fba52009-01-09 15:01:12 +01001654void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1655 int nelems, int direction)
1656{
1657 struct scatterlist *s;
FUJITA Tomonori88f39072009-05-27 09:43:03 +09001658 int mapped_ents = 0, i;
Joerg Roedela31fba52009-01-09 15:01:12 +01001659
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001660 if (unlikely(dma_debug_disabled()))
Joerg Roedela31fba52009-01-09 15:01:12 +01001661 return;
1662
1663 for_each_sg(sg, s, nelems, i) {
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001664
1665 struct dma_debug_entry ref = {
1666 .type = dma_debug_sg,
1667 .dev = dev,
Dan Williams0abdd7a2014-01-21 15:48:12 -08001668 .pfn = page_to_pfn(sg_page(s)),
1669 .offset = s->offset,
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001670 .dev_addr = sg_dma_address(s),
1671 .size = sg_dma_len(s),
1672 .direction = direction,
1673 .sg_call_ents = nelems,
1674 };
1675
FUJITA Tomonori88f39072009-05-27 09:43:03 +09001676 if (!i)
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001677 mapped_ents = get_nr_mapped_entries(dev, &ref);
FUJITA Tomonori88f39072009-05-27 09:43:03 +09001678
1679 if (i >= mapped_ents)
1680 break;
1681
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001682 check_sync(dev, &ref, true);
Joerg Roedela31fba52009-01-09 15:01:12 +01001683 }
1684}
1685EXPORT_SYMBOL(debug_dma_sync_sg_for_cpu);
1686
1687void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1688 int nelems, int direction)
1689{
1690 struct scatterlist *s;
FUJITA Tomonori88f39072009-05-27 09:43:03 +09001691 int mapped_ents = 0, i;
Joerg Roedela31fba52009-01-09 15:01:12 +01001692
Florian Fainelli01ce18b2014-12-10 15:41:23 -08001693 if (unlikely(dma_debug_disabled()))
Joerg Roedela31fba52009-01-09 15:01:12 +01001694 return;
1695
1696 for_each_sg(sg, s, nelems, i) {
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001697
1698 struct dma_debug_entry ref = {
1699 .type = dma_debug_sg,
1700 .dev = dev,
Dan Williams0abdd7a2014-01-21 15:48:12 -08001701 .pfn = page_to_pfn(sg_page(s)),
1702 .offset = s->offset,
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001703 .dev_addr = sg_dma_address(s),
1704 .size = sg_dma_len(s),
1705 .direction = direction,
1706 .sg_call_ents = nelems,
1707 };
FUJITA Tomonori88f39072009-05-27 09:43:03 +09001708 if (!i)
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001709 mapped_ents = get_nr_mapped_entries(dev, &ref);
FUJITA Tomonori88f39072009-05-27 09:43:03 +09001710
1711 if (i >= mapped_ents)
1712 break;
1713
Joerg Roedelaa010ef2009-06-12 15:25:06 +02001714 check_sync(dev, &ref, false);
Joerg Roedela31fba52009-01-09 15:01:12 +01001715 }
1716}
1717EXPORT_SYMBOL(debug_dma_sync_sg_for_device);
1718
Joerg Roedel1745de52009-05-22 21:49:51 +02001719static int __init dma_debug_driver_setup(char *str)
1720{
1721 int i;
1722
1723 for (i = 0; i < NAME_MAX_LEN - 1; ++i, ++str) {
1724 current_driver_name[i] = *str;
1725 if (*str == 0)
1726 break;
1727 }
1728
1729 if (current_driver_name[0])
Joerg Roedele7ed70e2009-06-08 15:39:24 +02001730 pr_info("DMA-API: enable driver filter for driver [%s]\n",
1731 current_driver_name);
Joerg Roedel1745de52009-05-22 21:49:51 +02001732
1733
1734 return 1;
1735}
1736__setup("dma_debug_driver=", dma_debug_driver_setup);