blob: bd203b61a72b32d1f20a35da758873ba61393f32 [file] [log] [blame]
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +10001/**************************************************************************
2 *
3 * Copyright 2006 Tungsten Graphics, Inc., Bismarck, ND., USA.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 *
27 **************************************************************************/
28
29/*
30 * Generic simple memory manager implementation. Intended to be used as a base
31 * class implementation for more advanced memory managers.
32 *
33 * Note that the algorithm used is quite simple and there might be substantial
34 * performance gains if a smarter free list is implemented. Currently it is just an
35 * unordered stack of free regions. This could easily be improved if an RB-tree
36 * is used instead. At least if we expect heavy fragmentation.
37 *
38 * Aligned allocations can also see improvement.
39 *
40 * Authors:
Jan Engelhardt96de0e22007-10-19 23:21:04 +020041 * Thomas Hellström <thomas-at-tungstengraphics-dot-com>
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +100042 */
43
David Howells760285e2012-10-02 18:01:07 +010044#include <drm/drmP.h>
45#include <drm/drm_mm.h>
Thomas Hellstrom1d584202007-01-08 22:25:47 +110046#include <linux/slab.h>
Dave Airliefa8a1232009-08-26 13:13:37 +100047#include <linux/seq_file.h>
Paul Gortmaker2d1a8a42011-08-30 18:16:33 -040048#include <linux/export.h>
Thomas Hellstrom1d584202007-01-08 22:25:47 +110049
Jerome Glisse249d6042009-04-08 17:11:16 +020050#define MM_UNUSED_TARGET 4
51
Jerome Glisse249d6042009-04-08 17:11:16 +020052static struct drm_mm_node *drm_mm_kmalloc(struct drm_mm *mm, int atomic)
Thomas Hellstrom1d584202007-01-08 22:25:47 +110053{
Dave Airlie55910512007-07-11 16:53:40 +100054 struct drm_mm_node *child;
Thomas Hellstrom1d584202007-01-08 22:25:47 +110055
Jerome Glisse249d6042009-04-08 17:11:16 +020056 if (atomic)
Daniel Vetter709ea972010-07-02 15:02:16 +010057 child = kzalloc(sizeof(*child), GFP_ATOMIC);
Jerome Glisse249d6042009-04-08 17:11:16 +020058 else
Daniel Vetter709ea972010-07-02 15:02:16 +010059 child = kzalloc(sizeof(*child), GFP_KERNEL);
Jerome Glisse249d6042009-04-08 17:11:16 +020060
61 if (unlikely(child == NULL)) {
62 spin_lock(&mm->unused_lock);
63 if (list_empty(&mm->unused_nodes))
64 child = NULL;
65 else {
66 child =
67 list_entry(mm->unused_nodes.next,
Daniel Vetterea7b1dd2011-02-18 17:59:12 +010068 struct drm_mm_node, node_list);
69 list_del(&child->node_list);
Jerome Glisse249d6042009-04-08 17:11:16 +020070 --mm->num_unused;
71 }
72 spin_unlock(&mm->unused_lock);
73 }
74 return child;
75}
76
Jerome Glissea698cf32009-11-13 20:56:58 +010077/* drm_mm_pre_get() - pre allocate drm_mm_node structure
78 * drm_mm: memory manager struct we are pre-allocating for
79 *
80 * Returns 0 on success or -ENOMEM if allocation fails.
81 */
Jerome Glisse249d6042009-04-08 17:11:16 +020082int drm_mm_pre_get(struct drm_mm *mm)
83{
84 struct drm_mm_node *node;
85
86 spin_lock(&mm->unused_lock);
87 while (mm->num_unused < MM_UNUSED_TARGET) {
88 spin_unlock(&mm->unused_lock);
Daniel Vetter709ea972010-07-02 15:02:16 +010089 node = kzalloc(sizeof(*node), GFP_KERNEL);
Jerome Glisse249d6042009-04-08 17:11:16 +020090 spin_lock(&mm->unused_lock);
91
92 if (unlikely(node == NULL)) {
93 int ret = (mm->num_unused < 2) ? -ENOMEM : 0;
94 spin_unlock(&mm->unused_lock);
95 return ret;
96 }
97 ++mm->num_unused;
Daniel Vetterea7b1dd2011-02-18 17:59:12 +010098 list_add_tail(&node->node_list, &mm->unused_nodes);
Jerome Glisse249d6042009-04-08 17:11:16 +020099 }
100 spin_unlock(&mm->unused_lock);
101 return 0;
102}
103EXPORT_SYMBOL(drm_mm_pre_get);
104
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100105static inline unsigned long drm_mm_hole_node_start(struct drm_mm_node *hole_node)
Jerome Glisse249d6042009-04-08 17:11:16 +0200106{
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100107 return hole_node->start + hole_node->size;
Thomas Hellstrom1d584202007-01-08 22:25:47 +1100108}
109
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100110static inline unsigned long drm_mm_hole_node_end(struct drm_mm_node *hole_node)
Thomas Hellstrom1d584202007-01-08 22:25:47 +1100111{
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100112 struct drm_mm_node *next_node =
113 list_entry(hole_node->node_list.next, struct drm_mm_node,
114 node_list);
Thomas Hellstrom1d584202007-01-08 22:25:47 +1100115
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100116 return next_node->start;
Thomas Hellstrom1d584202007-01-08 22:25:47 +1100117}
118
Daniel Vetter9fc935d2011-02-18 17:59:13 +0100119static void drm_mm_insert_helper(struct drm_mm_node *hole_node,
120 struct drm_mm_node *node,
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100121 unsigned long size, unsigned alignment,
122 unsigned long color)
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +1000123{
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100124 struct drm_mm *mm = hole_node->mm;
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100125 unsigned long hole_start = drm_mm_hole_node_start(hole_node);
126 unsigned long hole_end = drm_mm_hole_node_end(hole_node);
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100127 unsigned long adj_start = hole_start;
128 unsigned long adj_end = hole_end;
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100129
Daniel Vetterb0b7af12011-02-18 17:59:14 +0100130 BUG_ON(!hole_node->hole_follows || node->allocated);
131
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100132 if (mm->color_adjust)
133 mm->color_adjust(hole_node, color, &adj_start, &adj_end);
Thomas Hellstrom1d584202007-01-08 22:25:47 +1100134
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100135 if (alignment) {
136 unsigned tmp = adj_start % alignment;
137 if (tmp)
138 adj_start += alignment - tmp;
139 }
140
141 if (adj_start == hole_start) {
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100142 hole_node->hole_follows = 0;
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100143 list_del(&hole_node->hole_stack);
144 }
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +1000145
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100146 node->start = adj_start;
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100147 node->size = size;
148 node->mm = mm;
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100149 node->color = color;
Daniel Vetterb0b7af12011-02-18 17:59:14 +0100150 node->allocated = 1;
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100151
152 INIT_LIST_HEAD(&node->hole_stack);
153 list_add(&node->node_list, &hole_node->node_list);
154
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100155 BUG_ON(node->start + node->size > adj_end);
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100156
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100157 node->hole_follows = 0;
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100158 if (node->start + node->size < hole_end) {
159 list_add(&node->hole_stack, &mm->hole_stack);
160 node->hole_follows = 1;
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +1000161 }
Daniel Vetter9fc935d2011-02-18 17:59:13 +0100162}
163
Chris Wilson5973c7e2012-11-15 11:32:16 +0000164struct drm_mm_node *drm_mm_create_block(struct drm_mm *mm,
165 unsigned long start,
166 unsigned long size,
167 bool atomic)
168{
169 struct drm_mm_node *hole, *node;
170 unsigned long end = start + size;
171
172 list_for_each_entry(hole, &mm->hole_stack, hole_stack) {
173 unsigned long hole_start;
174 unsigned long hole_end;
175
176 BUG_ON(!hole->hole_follows);
177 hole_start = drm_mm_hole_node_start(hole);
178 hole_end = drm_mm_hole_node_end(hole);
179
180 if (hole_start > start || hole_end < end)
181 continue;
182
183 node = drm_mm_kmalloc(mm, atomic);
184 if (unlikely(node == NULL))
185 return NULL;
186
187 node->start = start;
188 node->size = size;
189 node->mm = mm;
190 node->allocated = 1;
191
192 INIT_LIST_HEAD(&node->hole_stack);
193 list_add(&node->node_list, &hole->node_list);
194
195 if (start == hole_start) {
196 hole->hole_follows = 0;
197 list_del_init(&hole->hole_stack);
198 }
199
200 node->hole_follows = 0;
201 if (end != hole_end) {
202 list_add(&node->hole_stack, &mm->hole_stack);
203 node->hole_follows = 1;
204 }
205
206 return node;
207 }
208
209 WARN(1, "no hole found for block 0x%lx + 0x%lx\n", start, size);
210 return NULL;
211}
212EXPORT_SYMBOL(drm_mm_create_block);
213
Daniel Vetter9fc935d2011-02-18 17:59:13 +0100214struct drm_mm_node *drm_mm_get_block_generic(struct drm_mm_node *hole_node,
215 unsigned long size,
216 unsigned alignment,
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100217 unsigned long color,
Daniel Vetter9fc935d2011-02-18 17:59:13 +0100218 int atomic)
219{
220 struct drm_mm_node *node;
221
Daniel Vetter9fc935d2011-02-18 17:59:13 +0100222 node = drm_mm_kmalloc(hole_node->mm, atomic);
223 if (unlikely(node == NULL))
224 return NULL;
225
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100226 drm_mm_insert_helper(hole_node, node, size, alignment, color);
Thomas Hellstrom1d584202007-01-08 22:25:47 +1100227
Chris Wilsone6c03c52009-05-22 14:14:22 +0100228 return node;
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +1000229}
Thomas Hellstrom89579f72009-06-17 12:29:56 +0200230EXPORT_SYMBOL(drm_mm_get_block_generic);
Jerome Glisse249d6042009-04-08 17:11:16 +0200231
Daniel Vetterb0b7af12011-02-18 17:59:14 +0100232/**
233 * Search for free space and insert a preallocated memory node. Returns
234 * -ENOSPC if no suitable free area is available. The preallocated memory node
235 * must be cleared.
236 */
237int drm_mm_insert_node(struct drm_mm *mm, struct drm_mm_node *node,
238 unsigned long size, unsigned alignment)
239{
240 struct drm_mm_node *hole_node;
241
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100242 hole_node = drm_mm_search_free(mm, size, alignment, false);
Daniel Vetterb0b7af12011-02-18 17:59:14 +0100243 if (!hole_node)
244 return -ENOSPC;
245
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100246 drm_mm_insert_helper(hole_node, node, size, alignment, 0);
Daniel Vetterb0b7af12011-02-18 17:59:14 +0100247
248 return 0;
249}
250EXPORT_SYMBOL(drm_mm_insert_node);
251
Daniel Vetter9fc935d2011-02-18 17:59:13 +0100252static void drm_mm_insert_helper_range(struct drm_mm_node *hole_node,
253 struct drm_mm_node *node,
254 unsigned long size, unsigned alignment,
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100255 unsigned long color,
Daniel Vetter9fc935d2011-02-18 17:59:13 +0100256 unsigned long start, unsigned long end)
Jerome Glissea2e68e92009-12-07 15:52:56 +0100257{
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100258 struct drm_mm *mm = hole_node->mm;
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100259 unsigned long hole_start = drm_mm_hole_node_start(hole_node);
260 unsigned long hole_end = drm_mm_hole_node_end(hole_node);
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100261 unsigned long adj_start = hole_start;
262 unsigned long adj_end = hole_end;
Jerome Glissea2e68e92009-12-07 15:52:56 +0100263
Daniel Vetterb0b7af12011-02-18 17:59:14 +0100264 BUG_ON(!hole_node->hole_follows || node->allocated);
265
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100266 if (mm->color_adjust)
267 mm->color_adjust(hole_node, color, &adj_start, &adj_end);
Jerome Glissea2e68e92009-12-07 15:52:56 +0100268
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100269 if (adj_start < start)
270 adj_start = start;
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100271
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100272 if (alignment) {
273 unsigned tmp = adj_start % alignment;
274 if (tmp)
275 adj_start += alignment - tmp;
Jerome Glissea2e68e92009-12-07 15:52:56 +0100276 }
277
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100278 if (adj_start == hole_start) {
279 hole_node->hole_follows = 0;
280 list_del(&hole_node->hole_stack);
281 }
282
283 node->start = adj_start;
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100284 node->size = size;
285 node->mm = mm;
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100286 node->color = color;
Daniel Vetterb0b7af12011-02-18 17:59:14 +0100287 node->allocated = 1;
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100288
289 INIT_LIST_HEAD(&node->hole_stack);
290 list_add(&node->node_list, &hole_node->node_list);
291
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100292 BUG_ON(node->start + node->size > adj_end);
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100293 BUG_ON(node->start + node->size > end);
294
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100295 node->hole_follows = 0;
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100296 if (node->start + node->size < hole_end) {
297 list_add(&node->hole_stack, &mm->hole_stack);
298 node->hole_follows = 1;
Jerome Glissea2e68e92009-12-07 15:52:56 +0100299 }
Daniel Vetter9fc935d2011-02-18 17:59:13 +0100300}
301
302struct drm_mm_node *drm_mm_get_block_range_generic(struct drm_mm_node *hole_node,
303 unsigned long size,
304 unsigned alignment,
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100305 unsigned long color,
Daniel Vetter9fc935d2011-02-18 17:59:13 +0100306 unsigned long start,
307 unsigned long end,
308 int atomic)
309{
310 struct drm_mm_node *node;
311
Daniel Vetter9fc935d2011-02-18 17:59:13 +0100312 node = drm_mm_kmalloc(hole_node->mm, atomic);
313 if (unlikely(node == NULL))
314 return NULL;
315
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100316 drm_mm_insert_helper_range(hole_node, node, size, alignment, color,
Daniel Vetter9fc935d2011-02-18 17:59:13 +0100317 start, end);
Jerome Glissea2e68e92009-12-07 15:52:56 +0100318
Jerome Glissea2e68e92009-12-07 15:52:56 +0100319 return node;
320}
321EXPORT_SYMBOL(drm_mm_get_block_range_generic);
322
Daniel Vetterb0b7af12011-02-18 17:59:14 +0100323/**
324 * Search for free space and insert a preallocated memory node. Returns
325 * -ENOSPC if no suitable free area is available. This is for range
326 * restricted allocations. The preallocated memory node must be cleared.
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +1000327 */
Daniel Vetterb0b7af12011-02-18 17:59:14 +0100328int drm_mm_insert_node_in_range(struct drm_mm *mm, struct drm_mm_node *node,
329 unsigned long size, unsigned alignment,
330 unsigned long start, unsigned long end)
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +1000331{
Daniel Vetterb0b7af12011-02-18 17:59:14 +0100332 struct drm_mm_node *hole_node;
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +1000333
Daniel Vetterb0b7af12011-02-18 17:59:14 +0100334 hole_node = drm_mm_search_free_in_range(mm, size, alignment,
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100335 start, end, false);
Daniel Vetterb0b7af12011-02-18 17:59:14 +0100336 if (!hole_node)
337 return -ENOSPC;
338
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100339 drm_mm_insert_helper_range(hole_node, node, size, alignment, 0,
Daniel Vetterb0b7af12011-02-18 17:59:14 +0100340 start, end);
341
342 return 0;
343}
344EXPORT_SYMBOL(drm_mm_insert_node_in_range);
345
346/**
347 * Remove a memory node from the allocator.
348 */
349void drm_mm_remove_node(struct drm_mm_node *node)
350{
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100351 struct drm_mm *mm = node->mm;
352 struct drm_mm_node *prev_node;
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +1000353
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100354 BUG_ON(node->scanned_block || node->scanned_prev_free
355 || node->scanned_next_free);
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +1000356
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100357 prev_node =
358 list_entry(node->node_list.prev, struct drm_mm_node, node_list);
Daniel Vetter709ea972010-07-02 15:02:16 +0100359
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100360 if (node->hole_follows) {
361 BUG_ON(drm_mm_hole_node_start(node)
362 == drm_mm_hole_node_end(node));
363 list_del(&node->hole_stack);
364 } else
365 BUG_ON(drm_mm_hole_node_start(node)
366 != drm_mm_hole_node_end(node));
367
368 if (!prev_node->hole_follows) {
369 prev_node->hole_follows = 1;
370 list_add(&prev_node->hole_stack, &mm->hole_stack);
371 } else
372 list_move(&prev_node->hole_stack, &mm->hole_stack);
373
374 list_del(&node->node_list);
Daniel Vetterb0b7af12011-02-18 17:59:14 +0100375 node->allocated = 0;
376}
377EXPORT_SYMBOL(drm_mm_remove_node);
378
379/*
380 * Remove a memory node from the allocator and free the allocated struct
381 * drm_mm_node. Only to be used on a struct drm_mm_node obtained by one of the
382 * drm_mm_get_block functions.
383 */
384void drm_mm_put_block(struct drm_mm_node *node)
385{
386
387 struct drm_mm *mm = node->mm;
388
389 drm_mm_remove_node(node);
390
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100391 spin_lock(&mm->unused_lock);
392 if (mm->num_unused < MM_UNUSED_TARGET) {
393 list_add(&node->node_list, &mm->unused_nodes);
394 ++mm->num_unused;
395 } else
396 kfree(node);
397 spin_unlock(&mm->unused_lock);
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +1000398}
Eric Anholt673a3942008-07-30 12:06:12 -0700399EXPORT_SYMBOL(drm_mm_put_block);
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +1000400
Daniel Vetter75214732010-08-26 21:44:17 +0200401static int check_free_hole(unsigned long start, unsigned long end,
402 unsigned long size, unsigned alignment)
Daniel Vetter7a6b2892010-07-02 15:02:15 +0100403{
Daniel Vetter75214732010-08-26 21:44:17 +0200404 if (end - start < size)
Daniel Vetter7a6b2892010-07-02 15:02:15 +0100405 return 0;
406
407 if (alignment) {
Daniel Vetter75214732010-08-26 21:44:17 +0200408 unsigned tmp = start % alignment;
Daniel Vetter7a6b2892010-07-02 15:02:15 +0100409 if (tmp)
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100410 start += alignment - tmp;
Daniel Vetter7a6b2892010-07-02 15:02:15 +0100411 }
412
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100413 return end >= start + size;
Daniel Vetter7a6b2892010-07-02 15:02:15 +0100414}
415
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100416struct drm_mm_node *drm_mm_search_free_generic(const struct drm_mm *mm,
417 unsigned long size,
418 unsigned alignment,
419 unsigned long color,
420 bool best_match)
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +1000421{
Dave Airlie55910512007-07-11 16:53:40 +1000422 struct drm_mm_node *entry;
423 struct drm_mm_node *best;
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +1000424 unsigned long best_size;
425
Daniel Vetter709ea972010-07-02 15:02:16 +0100426 BUG_ON(mm->scanned_blocks);
427
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +1000428 best = NULL;
429 best_size = ~0UL;
430
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100431 list_for_each_entry(entry, &mm->hole_stack, hole_stack) {
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100432 unsigned long adj_start = drm_mm_hole_node_start(entry);
433 unsigned long adj_end = drm_mm_hole_node_end(entry);
434
435 if (mm->color_adjust) {
436 mm->color_adjust(entry, color, &adj_start, &adj_end);
437 if (adj_end <= adj_start)
438 continue;
439 }
440
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100441 BUG_ON(!entry->hole_follows);
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100442 if (!check_free_hole(adj_start, adj_end, size, alignment))
Thomas Hellstrom1d584202007-01-08 22:25:47 +1100443 continue;
444
Daniel Vetter7a6b2892010-07-02 15:02:15 +0100445 if (!best_match)
446 return entry;
Thomas Hellstrom1d584202007-01-08 22:25:47 +1100447
Daniel Vetter7a6b2892010-07-02 15:02:15 +0100448 if (entry->size < best_size) {
449 best = entry;
450 best_size = entry->size;
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +1000451 }
452 }
453
454 return best;
455}
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100456EXPORT_SYMBOL(drm_mm_search_free_generic);
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +1000457
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100458struct drm_mm_node *drm_mm_search_free_in_range_generic(const struct drm_mm *mm,
459 unsigned long size,
460 unsigned alignment,
461 unsigned long color,
462 unsigned long start,
463 unsigned long end,
464 bool best_match)
Jerome Glissea2e68e92009-12-07 15:52:56 +0100465{
Jerome Glissea2e68e92009-12-07 15:52:56 +0100466 struct drm_mm_node *entry;
467 struct drm_mm_node *best;
468 unsigned long best_size;
Jerome Glissea2e68e92009-12-07 15:52:56 +0100469
Daniel Vetter709ea972010-07-02 15:02:16 +0100470 BUG_ON(mm->scanned_blocks);
471
Jerome Glissea2e68e92009-12-07 15:52:56 +0100472 best = NULL;
473 best_size = ~0UL;
474
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100475 list_for_each_entry(entry, &mm->hole_stack, hole_stack) {
476 unsigned long adj_start = drm_mm_hole_node_start(entry) < start ?
477 start : drm_mm_hole_node_start(entry);
478 unsigned long adj_end = drm_mm_hole_node_end(entry) > end ?
479 end : drm_mm_hole_node_end(entry);
Jerome Glissea2e68e92009-12-07 15:52:56 +0100480
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100481 BUG_ON(!entry->hole_follows);
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100482
483 if (mm->color_adjust) {
484 mm->color_adjust(entry, color, &adj_start, &adj_end);
485 if (adj_end <= adj_start)
486 continue;
487 }
488
Daniel Vetter75214732010-08-26 21:44:17 +0200489 if (!check_free_hole(adj_start, adj_end, size, alignment))
Daniel Vetter7a6b2892010-07-02 15:02:15 +0100490 continue;
Jerome Glissea2e68e92009-12-07 15:52:56 +0100491
Daniel Vetter7a6b2892010-07-02 15:02:15 +0100492 if (!best_match)
493 return entry;
Jerome Glissea2e68e92009-12-07 15:52:56 +0100494
Daniel Vetter7a6b2892010-07-02 15:02:15 +0100495 if (entry->size < best_size) {
496 best = entry;
497 best_size = entry->size;
Jerome Glissea2e68e92009-12-07 15:52:56 +0100498 }
499 }
500
501 return best;
502}
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100503EXPORT_SYMBOL(drm_mm_search_free_in_range_generic);
Jerome Glissea2e68e92009-12-07 15:52:56 +0100504
Daniel Vetter709ea972010-07-02 15:02:16 +0100505/**
Daniel Vetterb0b7af12011-02-18 17:59:14 +0100506 * Moves an allocation. To be used with embedded struct drm_mm_node.
507 */
508void drm_mm_replace_node(struct drm_mm_node *old, struct drm_mm_node *new)
509{
510 list_replace(&old->node_list, &new->node_list);
Daniel Vetter2bbd4492011-05-06 23:47:53 +0200511 list_replace(&old->hole_stack, &new->hole_stack);
Daniel Vetterb0b7af12011-02-18 17:59:14 +0100512 new->hole_follows = old->hole_follows;
513 new->mm = old->mm;
514 new->start = old->start;
515 new->size = old->size;
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100516 new->color = old->color;
Daniel Vetterb0b7af12011-02-18 17:59:14 +0100517
518 old->allocated = 0;
519 new->allocated = 1;
520}
521EXPORT_SYMBOL(drm_mm_replace_node);
522
523/**
Daniel Vetter709ea972010-07-02 15:02:16 +0100524 * Initializa lru scanning.
525 *
526 * This simply sets up the scanning routines with the parameters for the desired
527 * hole.
528 *
529 * Warning: As long as the scan list is non-empty, no other operations than
530 * adding/removing nodes to/from the scan list are allowed.
531 */
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100532void drm_mm_init_scan(struct drm_mm *mm,
533 unsigned long size,
534 unsigned alignment,
535 unsigned long color)
Daniel Vetter709ea972010-07-02 15:02:16 +0100536{
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100537 mm->scan_color = color;
Daniel Vetter709ea972010-07-02 15:02:16 +0100538 mm->scan_alignment = alignment;
539 mm->scan_size = size;
540 mm->scanned_blocks = 0;
541 mm->scan_hit_start = 0;
542 mm->scan_hit_size = 0;
Daniel Vetterd935cc62010-09-16 15:13:11 +0200543 mm->scan_check_range = 0;
Daniel Vetterae0cec22011-02-18 17:59:15 +0100544 mm->prev_scanned_node = NULL;
Daniel Vetter709ea972010-07-02 15:02:16 +0100545}
546EXPORT_SYMBOL(drm_mm_init_scan);
547
548/**
Daniel Vetterd935cc62010-09-16 15:13:11 +0200549 * Initializa lru scanning.
550 *
551 * This simply sets up the scanning routines with the parameters for the desired
552 * hole. This version is for range-restricted scans.
553 *
554 * Warning: As long as the scan list is non-empty, no other operations than
555 * adding/removing nodes to/from the scan list are allowed.
556 */
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100557void drm_mm_init_scan_with_range(struct drm_mm *mm,
558 unsigned long size,
Daniel Vetterd935cc62010-09-16 15:13:11 +0200559 unsigned alignment,
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100560 unsigned long color,
Daniel Vetterd935cc62010-09-16 15:13:11 +0200561 unsigned long start,
562 unsigned long end)
563{
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100564 mm->scan_color = color;
Daniel Vetterd935cc62010-09-16 15:13:11 +0200565 mm->scan_alignment = alignment;
566 mm->scan_size = size;
567 mm->scanned_blocks = 0;
568 mm->scan_hit_start = 0;
569 mm->scan_hit_size = 0;
570 mm->scan_start = start;
571 mm->scan_end = end;
572 mm->scan_check_range = 1;
Daniel Vetterae0cec22011-02-18 17:59:15 +0100573 mm->prev_scanned_node = NULL;
Daniel Vetterd935cc62010-09-16 15:13:11 +0200574}
575EXPORT_SYMBOL(drm_mm_init_scan_with_range);
576
577/**
Daniel Vetter709ea972010-07-02 15:02:16 +0100578 * Add a node to the scan list that might be freed to make space for the desired
579 * hole.
580 *
581 * Returns non-zero, if a hole has been found, zero otherwise.
582 */
583int drm_mm_scan_add_block(struct drm_mm_node *node)
584{
585 struct drm_mm *mm = node->mm;
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100586 struct drm_mm_node *prev_node;
587 unsigned long hole_start, hole_end;
Daniel Vetterd935cc62010-09-16 15:13:11 +0200588 unsigned long adj_start;
589 unsigned long adj_end;
Daniel Vetter709ea972010-07-02 15:02:16 +0100590
591 mm->scanned_blocks++;
592
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100593 BUG_ON(node->scanned_block);
Daniel Vetter709ea972010-07-02 15:02:16 +0100594 node->scanned_block = 1;
Daniel Vetter709ea972010-07-02 15:02:16 +0100595
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100596 prev_node = list_entry(node->node_list.prev, struct drm_mm_node,
597 node_list);
Daniel Vetter709ea972010-07-02 15:02:16 +0100598
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100599 node->scanned_preceeds_hole = prev_node->hole_follows;
600 prev_node->hole_follows = 1;
601 list_del(&node->node_list);
602 node->node_list.prev = &prev_node->node_list;
Daniel Vetterae0cec22011-02-18 17:59:15 +0100603 node->node_list.next = &mm->prev_scanned_node->node_list;
604 mm->prev_scanned_node = node;
Daniel Vetter709ea972010-07-02 15:02:16 +0100605
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100606 hole_start = drm_mm_hole_node_start(prev_node);
607 hole_end = drm_mm_hole_node_end(prev_node);
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100608
609 adj_start = hole_start;
610 adj_end = hole_end;
611
612 if (mm->color_adjust)
613 mm->color_adjust(prev_node, mm->scan_color, &adj_start, &adj_end);
614
Daniel Vetterd935cc62010-09-16 15:13:11 +0200615 if (mm->scan_check_range) {
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100616 if (adj_start < mm->scan_start)
617 adj_start = mm->scan_start;
618 if (adj_end > mm->scan_end)
619 adj_end = mm->scan_end;
Daniel Vetterd935cc62010-09-16 15:13:11 +0200620 }
621
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100622 if (check_free_hole(adj_start, adj_end,
Daniel Vetter75214732010-08-26 21:44:17 +0200623 mm->scan_size, mm->scan_alignment)) {
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100624 mm->scan_hit_start = hole_start;
625 mm->scan_hit_size = hole_end;
Daniel Vetter709ea972010-07-02 15:02:16 +0100626
627 return 1;
628 }
629
630 return 0;
631}
632EXPORT_SYMBOL(drm_mm_scan_add_block);
633
634/**
635 * Remove a node from the scan list.
636 *
637 * Nodes _must_ be removed in the exact same order from the scan list as they
638 * have been added, otherwise the internal state of the memory manager will be
639 * corrupted.
640 *
641 * When the scan list is empty, the selected memory nodes can be freed. An
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300642 * immediately following drm_mm_search_free with best_match = 0 will then return
Daniel Vetter709ea972010-07-02 15:02:16 +0100643 * the just freed block (because its at the top of the free_stack list).
644 *
645 * Returns one if this block should be evicted, zero otherwise. Will always
646 * return zero when no hole has been found.
647 */
648int drm_mm_scan_remove_block(struct drm_mm_node *node)
649{
650 struct drm_mm *mm = node->mm;
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100651 struct drm_mm_node *prev_node;
Daniel Vetter709ea972010-07-02 15:02:16 +0100652
653 mm->scanned_blocks--;
654
655 BUG_ON(!node->scanned_block);
656 node->scanned_block = 0;
Daniel Vetter709ea972010-07-02 15:02:16 +0100657
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100658 prev_node = list_entry(node->node_list.prev, struct drm_mm_node,
659 node_list);
Daniel Vetter709ea972010-07-02 15:02:16 +0100660
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100661 prev_node->hole_follows = node->scanned_preceeds_hole;
662 INIT_LIST_HEAD(&node->node_list);
663 list_add(&node->node_list, &prev_node->node_list);
Daniel Vetter709ea972010-07-02 15:02:16 +0100664
665 /* Only need to check for containement because start&size for the
666 * complete resulting free block (not just the desired part) is
667 * stored. */
668 if (node->start >= mm->scan_hit_start &&
669 node->start + node->size
670 <= mm->scan_hit_start + mm->scan_hit_size) {
671 return 1;
672 }
673
674 return 0;
675}
676EXPORT_SYMBOL(drm_mm_scan_remove_block);
677
Dave Airlie55910512007-07-11 16:53:40 +1000678int drm_mm_clean(struct drm_mm * mm)
Thomas Hellstrom1d584202007-01-08 22:25:47 +1100679{
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100680 struct list_head *head = &mm->head_node.node_list;
Thomas Hellstrom1d584202007-01-08 22:25:47 +1100681
682 return (head->next->next == head);
683}
Jerome Glisse249d6042009-04-08 17:11:16 +0200684EXPORT_SYMBOL(drm_mm_clean);
Thomas Hellstrom1d584202007-01-08 22:25:47 +1100685
Dave Airlie55910512007-07-11 16:53:40 +1000686int drm_mm_init(struct drm_mm * mm, unsigned long start, unsigned long size)
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +1000687{
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100688 INIT_LIST_HEAD(&mm->hole_stack);
Jerome Glisse249d6042009-04-08 17:11:16 +0200689 INIT_LIST_HEAD(&mm->unused_nodes);
690 mm->num_unused = 0;
Daniel Vetter709ea972010-07-02 15:02:16 +0100691 mm->scanned_blocks = 0;
Jerome Glisse249d6042009-04-08 17:11:16 +0200692 spin_lock_init(&mm->unused_lock);
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +1000693
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100694 /* Clever trick to avoid a special case in the free hole tracking. */
695 INIT_LIST_HEAD(&mm->head_node.node_list);
696 INIT_LIST_HEAD(&mm->head_node.hole_stack);
697 mm->head_node.hole_follows = 1;
698 mm->head_node.scanned_block = 0;
699 mm->head_node.scanned_prev_free = 0;
700 mm->head_node.scanned_next_free = 0;
701 mm->head_node.mm = mm;
702 mm->head_node.start = start + size;
703 mm->head_node.size = start - mm->head_node.start;
704 list_add_tail(&mm->head_node.hole_stack, &mm->hole_stack);
705
Chris Wilson6b9d89b2012-07-10 11:15:23 +0100706 mm->color_adjust = NULL;
707
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100708 return 0;
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +1000709}
Eric Anholt673a3942008-07-30 12:06:12 -0700710EXPORT_SYMBOL(drm_mm_init);
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +1000711
Dave Airlie55910512007-07-11 16:53:40 +1000712void drm_mm_takedown(struct drm_mm * mm)
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +1000713{
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100714 struct drm_mm_node *entry, *next;
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +1000715
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100716 if (!list_empty(&mm->head_node.node_list)) {
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +1000717 DRM_ERROR("Memory manager not clean. Delaying takedown\n");
718 return;
719 }
720
Jerome Glisse249d6042009-04-08 17:11:16 +0200721 spin_lock(&mm->unused_lock);
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100722 list_for_each_entry_safe(entry, next, &mm->unused_nodes, node_list) {
723 list_del(&entry->node_list);
Jerome Glisse249d6042009-04-08 17:11:16 +0200724 kfree(entry);
725 --mm->num_unused;
726 }
727 spin_unlock(&mm->unused_lock);
728
729 BUG_ON(mm->num_unused != 0);
Thomas Hellstrom3a1bd922006-08-07 21:30:28 +1000730}
Dave Airlief453ba02008-11-07 14:05:41 -0800731EXPORT_SYMBOL(drm_mm_takedown);
Dave Airliefa8a1232009-08-26 13:13:37 +1000732
Jerome Glisse99d7e482009-12-09 21:55:09 +0100733void drm_mm_debug_table(struct drm_mm *mm, const char *prefix)
734{
735 struct drm_mm_node *entry;
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100736 unsigned long total_used = 0, total_free = 0, total = 0;
737 unsigned long hole_start, hole_end, hole_size;
Jerome Glisse99d7e482009-12-09 21:55:09 +0100738
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100739 hole_start = drm_mm_hole_node_start(&mm->head_node);
740 hole_end = drm_mm_hole_node_end(&mm->head_node);
741 hole_size = hole_end - hole_start;
742 if (hole_size)
743 printk(KERN_DEBUG "%s 0x%08lx-0x%08lx: %8lu: free\n",
744 prefix, hole_start, hole_end,
745 hole_size);
746 total_free += hole_size;
747
748 drm_mm_for_each_node(entry, mm) {
749 printk(KERN_DEBUG "%s 0x%08lx-0x%08lx: %8lu: used\n",
Jerome Glisse99d7e482009-12-09 21:55:09 +0100750 prefix, entry->start, entry->start + entry->size,
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100751 entry->size);
752 total_used += entry->size;
753
754 if (entry->hole_follows) {
755 hole_start = drm_mm_hole_node_start(entry);
756 hole_end = drm_mm_hole_node_end(entry);
757 hole_size = hole_end - hole_start;
758 printk(KERN_DEBUG "%s 0x%08lx-0x%08lx: %8lu: free\n",
759 prefix, hole_start, hole_end,
760 hole_size);
761 total_free += hole_size;
762 }
Jerome Glisse99d7e482009-12-09 21:55:09 +0100763 }
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100764 total = total_free + total_used;
765
766 printk(KERN_DEBUG "%s total: %lu, used %lu free %lu\n", prefix, total,
Jerome Glisse99d7e482009-12-09 21:55:09 +0100767 total_used, total_free);
768}
769EXPORT_SYMBOL(drm_mm_debug_table);
770
Dave Airliefa8a1232009-08-26 13:13:37 +1000771#if defined(CONFIG_DEBUG_FS)
772int drm_mm_dump_table(struct seq_file *m, struct drm_mm *mm)
773{
774 struct drm_mm_node *entry;
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100775 unsigned long total_used = 0, total_free = 0, total = 0;
776 unsigned long hole_start, hole_end, hole_size;
Dave Airliefa8a1232009-08-26 13:13:37 +1000777
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100778 hole_start = drm_mm_hole_node_start(&mm->head_node);
779 hole_end = drm_mm_hole_node_end(&mm->head_node);
780 hole_size = hole_end - hole_start;
781 if (hole_size)
782 seq_printf(m, "0x%08lx-0x%08lx: 0x%08lx: free\n",
783 hole_start, hole_end, hole_size);
784 total_free += hole_size;
785
786 drm_mm_for_each_node(entry, mm) {
787 seq_printf(m, "0x%08lx-0x%08lx: 0x%08lx: used\n",
788 entry->start, entry->start + entry->size,
789 entry->size);
790 total_used += entry->size;
791 if (entry->hole_follows) {
Daniel Vetter2bbd4492011-05-06 23:47:53 +0200792 hole_start = drm_mm_hole_node_start(entry);
793 hole_end = drm_mm_hole_node_end(entry);
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100794 hole_size = hole_end - hole_start;
795 seq_printf(m, "0x%08lx-0x%08lx: 0x%08lx: free\n",
796 hole_start, hole_end, hole_size);
797 total_free += hole_size;
798 }
Dave Airliefa8a1232009-08-26 13:13:37 +1000799 }
Daniel Vetterea7b1dd2011-02-18 17:59:12 +0100800 total = total_free + total_used;
801
802 seq_printf(m, "total: %lu, used %lu free %lu\n", total, total_used, total_free);
Dave Airliefa8a1232009-08-26 13:13:37 +1000803 return 0;
804}
805EXPORT_SYMBOL(drm_mm_dump_table);
806#endif