blob: 1e5b17dc7e3d5c6da19bdb2376e7a0a120cbc722 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2001 Momchil Velikov
3 * Portions Copyright (C) 2001 Christoph Hellwig
Christoph Lameter201b6262005-09-06 15:16:46 -07004 * Copyright (C) 2005 SGI, Christoph Lameter <clameter@sgi.com>
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as
8 * published by the Free Software Foundation; either version 2, or (at
9 * your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19 */
20
21#include <linux/errno.h>
22#include <linux/init.h>
23#include <linux/kernel.h>
24#include <linux/module.h>
25#include <linux/radix-tree.h>
26#include <linux/percpu.h>
27#include <linux/slab.h>
28#include <linux/notifier.h>
29#include <linux/cpu.h>
30#include <linux/gfp.h>
31#include <linux/string.h>
32#include <linux/bitops.h>
33
34
35#ifdef __KERNEL__
36#define RADIX_TREE_MAP_SHIFT 6
37#else
38#define RADIX_TREE_MAP_SHIFT 3 /* For more stressful testing */
39#endif
40#define RADIX_TREE_TAGS 2
41
42#define RADIX_TREE_MAP_SIZE (1UL << RADIX_TREE_MAP_SHIFT)
43#define RADIX_TREE_MAP_MASK (RADIX_TREE_MAP_SIZE-1)
44
45#define RADIX_TREE_TAG_LONGS \
46 ((RADIX_TREE_MAP_SIZE + BITS_PER_LONG - 1) / BITS_PER_LONG)
47
48struct radix_tree_node {
49 unsigned int count;
50 void *slots[RADIX_TREE_MAP_SIZE];
51 unsigned long tags[RADIX_TREE_TAGS][RADIX_TREE_TAG_LONGS];
52};
53
54struct radix_tree_path {
Christoph Lameter201b6262005-09-06 15:16:46 -070055 struct radix_tree_node *node;
Linus Torvalds1da177e2005-04-16 15:20:36 -070056 int offset;
57};
58
59#define RADIX_TREE_INDEX_BITS (8 /* CHAR_BIT */ * sizeof(unsigned long))
60#define RADIX_TREE_MAX_PATH (RADIX_TREE_INDEX_BITS/RADIX_TREE_MAP_SHIFT + 2)
61
Christoph Lameter6c036522005-07-07 17:56:59 -070062static unsigned long height_to_maxindex[RADIX_TREE_MAX_PATH] __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -070063
64/*
65 * Radix tree node cache.
66 */
67static kmem_cache_t *radix_tree_node_cachep;
68
69/*
70 * Per-cpu pool of preloaded nodes
71 */
72struct radix_tree_preload {
73 int nr;
74 struct radix_tree_node *nodes[RADIX_TREE_MAX_PATH];
75};
76DEFINE_PER_CPU(struct radix_tree_preload, radix_tree_preloads) = { 0, };
77
78/*
79 * This assumes that the caller has performed appropriate preallocation, and
80 * that the caller has pinned this thread of control to the current CPU.
81 */
82static struct radix_tree_node *
83radix_tree_node_alloc(struct radix_tree_root *root)
84{
85 struct radix_tree_node *ret;
86
87 ret = kmem_cache_alloc(radix_tree_node_cachep, root->gfp_mask);
88 if (ret == NULL && !(root->gfp_mask & __GFP_WAIT)) {
89 struct radix_tree_preload *rtp;
90
91 rtp = &__get_cpu_var(radix_tree_preloads);
92 if (rtp->nr) {
93 ret = rtp->nodes[rtp->nr - 1];
94 rtp->nodes[rtp->nr - 1] = NULL;
95 rtp->nr--;
96 }
97 }
98 return ret;
99}
100
101static inline void
102radix_tree_node_free(struct radix_tree_node *node)
103{
104 kmem_cache_free(radix_tree_node_cachep, node);
105}
106
107/*
108 * Load up this CPU's radix_tree_node buffer with sufficient objects to
109 * ensure that the addition of a single element in the tree cannot fail. On
110 * success, return zero, with preemption disabled. On error, return -ENOMEM
111 * with preemption not disabled.
112 */
Al Virodd0fc662005-10-07 07:46:04 +0100113int radix_tree_preload(gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114{
115 struct radix_tree_preload *rtp;
116 struct radix_tree_node *node;
117 int ret = -ENOMEM;
118
119 preempt_disable();
120 rtp = &__get_cpu_var(radix_tree_preloads);
121 while (rtp->nr < ARRAY_SIZE(rtp->nodes)) {
122 preempt_enable();
123 node = kmem_cache_alloc(radix_tree_node_cachep, gfp_mask);
124 if (node == NULL)
125 goto out;
126 preempt_disable();
127 rtp = &__get_cpu_var(radix_tree_preloads);
128 if (rtp->nr < ARRAY_SIZE(rtp->nodes))
129 rtp->nodes[rtp->nr++] = node;
130 else
131 kmem_cache_free(radix_tree_node_cachep, node);
132 }
133 ret = 0;
134out:
135 return ret;
136}
137
138static inline void tag_set(struct radix_tree_node *node, int tag, int offset)
139{
Nick Piggind5274262006-01-08 01:01:41 -0800140 __set_bit(offset, node->tags[tag]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700141}
142
143static inline void tag_clear(struct radix_tree_node *node, int tag, int offset)
144{
Nick Piggind5274262006-01-08 01:01:41 -0800145 __clear_bit(offset, node->tags[tag]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146}
147
148static inline int tag_get(struct radix_tree_node *node, int tag, int offset)
149{
Nick Piggind5274262006-01-08 01:01:41 -0800150 return test_bit(offset, node->tags[tag]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151}
152
153/*
Nick Piggin6e954b92006-01-08 01:01:40 -0800154 * Returns 1 if any slot in the node has this tag set.
155 * Otherwise returns 0.
156 */
157static inline int any_tag_set(struct radix_tree_node *node, int tag)
158{
159 int idx;
160 for (idx = 0; idx < RADIX_TREE_TAG_LONGS; idx++) {
161 if (node->tags[tag][idx])
162 return 1;
163 }
164 return 0;
165}
166
167/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168 * Return the maximum key which can be store into a
169 * radix tree with height HEIGHT.
170 */
171static inline unsigned long radix_tree_maxindex(unsigned int height)
172{
173 return height_to_maxindex[height];
174}
175
176/*
177 * Extend a radix tree so it can store key @index.
178 */
179static int radix_tree_extend(struct radix_tree_root *root, unsigned long index)
180{
181 struct radix_tree_node *node;
182 unsigned int height;
183 char tags[RADIX_TREE_TAGS];
184 int tag;
185
186 /* Figure out what the height should be. */
187 height = root->height + 1;
188 while (index > radix_tree_maxindex(height))
189 height++;
190
191 if (root->rnode == NULL) {
192 root->height = height;
193 goto out;
194 }
195
196 /*
197 * Prepare the tag status of the top-level node for propagation
198 * into the newly-pushed top-level node(s)
199 */
200 for (tag = 0; tag < RADIX_TREE_TAGS; tag++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201 tags[tag] = 0;
Nick Piggin6e954b92006-01-08 01:01:40 -0800202 if (any_tag_set(root->rnode, tag))
203 tags[tag] = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204 }
205
206 do {
207 if (!(node = radix_tree_node_alloc(root)))
208 return -ENOMEM;
209
210 /* Increase the height. */
211 node->slots[0] = root->rnode;
212
213 /* Propagate the aggregated tag info into the new root */
214 for (tag = 0; tag < RADIX_TREE_TAGS; tag++) {
215 if (tags[tag])
216 tag_set(node, tag, 0);
217 }
218
219 node->count = 1;
220 root->rnode = node;
221 root->height++;
222 } while (height > root->height);
223out:
224 return 0;
225}
226
227/**
228 * radix_tree_insert - insert into a radix tree
229 * @root: radix tree root
230 * @index: index key
231 * @item: item to insert
232 *
233 * Insert an item into the radix tree at position @index.
234 */
235int radix_tree_insert(struct radix_tree_root *root,
236 unsigned long index, void *item)
237{
Christoph Lameter201b6262005-09-06 15:16:46 -0700238 struct radix_tree_node *node = NULL, *slot;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239 unsigned int height, shift;
240 int offset;
241 int error;
242
243 /* Make sure the tree is high enough. */
244 if ((!index && !root->rnode) ||
245 index > radix_tree_maxindex(root->height)) {
246 error = radix_tree_extend(root, index);
247 if (error)
248 return error;
249 }
250
Christoph Lameter201b6262005-09-06 15:16:46 -0700251 slot = root->rnode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700252 height = root->height;
253 shift = (height-1) * RADIX_TREE_MAP_SHIFT;
254
255 offset = 0; /* uninitialised var warning */
Nick Piggina5f51c92006-01-08 01:01:41 -0800256 do {
Christoph Lameter201b6262005-09-06 15:16:46 -0700257 if (slot == NULL) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258 /* Have to add a child node. */
Christoph Lameter201b6262005-09-06 15:16:46 -0700259 if (!(slot = radix_tree_node_alloc(root)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700260 return -ENOMEM;
Christoph Lameter201b6262005-09-06 15:16:46 -0700261 if (node) {
262 node->slots[offset] = slot;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263 node->count++;
Christoph Lameter201b6262005-09-06 15:16:46 -0700264 } else
265 root->rnode = slot;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266 }
267
268 /* Go a level down */
269 offset = (index >> shift) & RADIX_TREE_MAP_MASK;
Christoph Lameter201b6262005-09-06 15:16:46 -0700270 node = slot;
271 slot = node->slots[offset];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272 shift -= RADIX_TREE_MAP_SHIFT;
273 height--;
Nick Piggina5f51c92006-01-08 01:01:41 -0800274 } while (height > 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275
Christoph Lameter201b6262005-09-06 15:16:46 -0700276 if (slot != NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700277 return -EEXIST;
Christoph Lameter201b6262005-09-06 15:16:46 -0700278
Nick Piggina5f51c92006-01-08 01:01:41 -0800279 BUG_ON(!node);
280 node->count++;
281 node->slots[offset] = item;
282 BUG_ON(tag_get(node, 0, offset));
283 BUG_ON(tag_get(node, 1, offset));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284
Linus Torvalds1da177e2005-04-16 15:20:36 -0700285 return 0;
286}
287EXPORT_SYMBOL(radix_tree_insert);
288
Hans Reisera4331362005-11-07 00:59:29 -0800289static inline void **__lookup_slot(struct radix_tree_root *root,
290 unsigned long index)
291{
292 unsigned int height, shift;
293 struct radix_tree_node **slot;
294
295 height = root->height;
296 if (index > radix_tree_maxindex(height))
297 return NULL;
298
299 shift = (height-1) * RADIX_TREE_MAP_SHIFT;
300 slot = &root->rnode;
301
302 while (height > 0) {
303 if (*slot == NULL)
304 return NULL;
305
306 slot = (struct radix_tree_node **)
307 ((*slot)->slots +
308 ((index >> shift) & RADIX_TREE_MAP_MASK));
309 shift -= RADIX_TREE_MAP_SHIFT;
310 height--;
311 }
312
313 return (void **)slot;
314}
315
316/**
317 * radix_tree_lookup_slot - lookup a slot in a radix tree
318 * @root: radix tree root
319 * @index: index key
320 *
321 * Lookup the slot corresponding to the position @index in the radix tree
322 * @root. This is useful for update-if-exists operations.
323 */
324void **radix_tree_lookup_slot(struct radix_tree_root *root, unsigned long index)
325{
326 return __lookup_slot(root, index);
327}
328EXPORT_SYMBOL(radix_tree_lookup_slot);
329
Linus Torvalds1da177e2005-04-16 15:20:36 -0700330/**
331 * radix_tree_lookup - perform lookup operation on a radix tree
332 * @root: radix tree root
333 * @index: index key
334 *
335 * Lookup the item at the position @index in the radix tree @root.
336 */
337void *radix_tree_lookup(struct radix_tree_root *root, unsigned long index)
338{
Hans Reisera4331362005-11-07 00:59:29 -0800339 void **slot;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340
Hans Reisera4331362005-11-07 00:59:29 -0800341 slot = __lookup_slot(root, index);
342 return slot != NULL ? *slot : NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343}
344EXPORT_SYMBOL(radix_tree_lookup);
345
346/**
347 * radix_tree_tag_set - set a tag on a radix tree node
348 * @root: radix tree root
349 * @index: index key
350 * @tag: tag index
351 *
352 * Set the search tag corresponging to @index in the radix tree. From
353 * the root all the way down to the leaf node.
354 *
355 * Returns the address of the tagged item. Setting a tag on a not-present
356 * item is a bug.
357 */
358void *radix_tree_tag_set(struct radix_tree_root *root,
359 unsigned long index, int tag)
360{
361 unsigned int height, shift;
Christoph Lameter201b6262005-09-06 15:16:46 -0700362 struct radix_tree_node *slot;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363
364 height = root->height;
365 if (index > radix_tree_maxindex(height))
366 return NULL;
367
368 shift = (height - 1) * RADIX_TREE_MAP_SHIFT;
Christoph Lameter201b6262005-09-06 15:16:46 -0700369 slot = root->rnode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370
371 while (height > 0) {
372 int offset;
373
374 offset = (index >> shift) & RADIX_TREE_MAP_MASK;
Nick Piggind5274262006-01-08 01:01:41 -0800375 if (!tag_get(slot, tag, offset))
376 tag_set(slot, tag, offset);
Christoph Lameter201b6262005-09-06 15:16:46 -0700377 slot = slot->slots[offset];
378 BUG_ON(slot == NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379 shift -= RADIX_TREE_MAP_SHIFT;
380 height--;
381 }
382
Christoph Lameter201b6262005-09-06 15:16:46 -0700383 return slot;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384}
385EXPORT_SYMBOL(radix_tree_tag_set);
386
387/**
388 * radix_tree_tag_clear - clear a tag on a radix tree node
389 * @root: radix tree root
390 * @index: index key
391 * @tag: tag index
392 *
393 * Clear the search tag corresponging to @index in the radix tree. If
394 * this causes the leaf node to have no tags set then clear the tag in the
395 * next-to-leaf node, etc.
396 *
397 * Returns the address of the tagged item on success, else NULL. ie:
398 * has the same return value and semantics as radix_tree_lookup().
399 */
400void *radix_tree_tag_clear(struct radix_tree_root *root,
401 unsigned long index, int tag)
402{
403 struct radix_tree_path path[RADIX_TREE_MAX_PATH], *pathp = path;
Christoph Lameter201b6262005-09-06 15:16:46 -0700404 struct radix_tree_node *slot;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700405 unsigned int height, shift;
406 void *ret = NULL;
407
408 height = root->height;
409 if (index > radix_tree_maxindex(height))
410 goto out;
411
412 shift = (height - 1) * RADIX_TREE_MAP_SHIFT;
413 pathp->node = NULL;
Christoph Lameter201b6262005-09-06 15:16:46 -0700414 slot = root->rnode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415
416 while (height > 0) {
417 int offset;
418
Christoph Lameter201b6262005-09-06 15:16:46 -0700419 if (slot == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420 goto out;
421
422 offset = (index >> shift) & RADIX_TREE_MAP_MASK;
423 pathp[1].offset = offset;
Christoph Lameter201b6262005-09-06 15:16:46 -0700424 pathp[1].node = slot;
425 slot = slot->slots[offset];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426 pathp++;
427 shift -= RADIX_TREE_MAP_SHIFT;
428 height--;
429 }
430
Christoph Lameter201b6262005-09-06 15:16:46 -0700431 ret = slot;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432 if (ret == NULL)
433 goto out;
434
435 do {
Nick Piggind5274262006-01-08 01:01:41 -0800436 if (!tag_get(pathp->node, tag, pathp->offset))
437 goto out;
Christoph Lameter201b6262005-09-06 15:16:46 -0700438 tag_clear(pathp->node, tag, pathp->offset);
Nick Piggin6e954b92006-01-08 01:01:40 -0800439 if (any_tag_set(pathp->node, tag))
440 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700441 pathp--;
Christoph Lameter201b6262005-09-06 15:16:46 -0700442 } while (pathp->node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443out:
444 return ret;
445}
446EXPORT_SYMBOL(radix_tree_tag_clear);
447
448#ifndef __KERNEL__ /* Only the test harness uses this at present */
449/**
Marcelo Tosatti32605a12005-09-06 15:16:48 -0700450 * radix_tree_tag_get - get a tag on a radix tree node
451 * @root: radix tree root
452 * @index: index key
453 * @tag: tag index
Linus Torvalds1da177e2005-04-16 15:20:36 -0700454 *
Marcelo Tosatti32605a12005-09-06 15:16:48 -0700455 * Return values:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700456 *
Marcelo Tosatti32605a12005-09-06 15:16:48 -0700457 * 0: tag not present
458 * 1: tag present, set
459 * -1: tag present, unset
Linus Torvalds1da177e2005-04-16 15:20:36 -0700460 */
461int radix_tree_tag_get(struct radix_tree_root *root,
462 unsigned long index, int tag)
463{
464 unsigned int height, shift;
Christoph Lameter201b6262005-09-06 15:16:46 -0700465 struct radix_tree_node *slot;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700466 int saw_unset_tag = 0;
467
468 height = root->height;
469 if (index > radix_tree_maxindex(height))
470 return 0;
471
472 shift = (height - 1) * RADIX_TREE_MAP_SHIFT;
Christoph Lameter201b6262005-09-06 15:16:46 -0700473 slot = root->rnode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474
475 for ( ; ; ) {
476 int offset;
477
Christoph Lameter201b6262005-09-06 15:16:46 -0700478 if (slot == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700479 return 0;
480
481 offset = (index >> shift) & RADIX_TREE_MAP_MASK;
482
483 /*
484 * This is just a debug check. Later, we can bale as soon as
485 * we see an unset tag.
486 */
Christoph Lameter201b6262005-09-06 15:16:46 -0700487 if (!tag_get(slot, tag, offset))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488 saw_unset_tag = 1;
489 if (height == 1) {
Christoph Lameter201b6262005-09-06 15:16:46 -0700490 int ret = tag_get(slot, tag, offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700491
492 BUG_ON(ret && saw_unset_tag);
Marcelo Tosatti32605a12005-09-06 15:16:48 -0700493 return ret ? 1 : -1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700494 }
Christoph Lameter201b6262005-09-06 15:16:46 -0700495 slot = slot->slots[offset];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700496 shift -= RADIX_TREE_MAP_SHIFT;
497 height--;
498 }
499}
500EXPORT_SYMBOL(radix_tree_tag_get);
501#endif
502
503static unsigned int
504__lookup(struct radix_tree_root *root, void **results, unsigned long index,
505 unsigned int max_items, unsigned long *next_index)
506{
507 unsigned int nr_found = 0;
Christoph Lameter201b6262005-09-06 15:16:46 -0700508 unsigned int shift, height;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700509 struct radix_tree_node *slot;
Christoph Lameter201b6262005-09-06 15:16:46 -0700510 unsigned long i;
511
512 height = root->height;
513 if (height == 0)
514 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700515
516 shift = (height-1) * RADIX_TREE_MAP_SHIFT;
517 slot = root->rnode;
518
Christoph Lameter201b6262005-09-06 15:16:46 -0700519 for ( ; height > 1; height--) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700520
Christoph Lameter201b6262005-09-06 15:16:46 -0700521 for (i = (index >> shift) & RADIX_TREE_MAP_MASK ;
522 i < RADIX_TREE_MAP_SIZE; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523 if (slot->slots[i] != NULL)
524 break;
525 index &= ~((1UL << shift) - 1);
526 index += 1UL << shift;
527 if (index == 0)
528 goto out; /* 32-bit wraparound */
529 }
530 if (i == RADIX_TREE_MAP_SIZE)
531 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700532
Linus Torvalds1da177e2005-04-16 15:20:36 -0700533 shift -= RADIX_TREE_MAP_SHIFT;
534 slot = slot->slots[i];
535 }
Christoph Lameter201b6262005-09-06 15:16:46 -0700536
537 /* Bottom level: grab some items */
538 for (i = index & RADIX_TREE_MAP_MASK; i < RADIX_TREE_MAP_SIZE; i++) {
539 index++;
540 if (slot->slots[i]) {
541 results[nr_found++] = slot->slots[i];
542 if (nr_found == max_items)
543 goto out;
544 }
545 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700546out:
547 *next_index = index;
548 return nr_found;
549}
550
551/**
552 * radix_tree_gang_lookup - perform multiple lookup on a radix tree
553 * @root: radix tree root
554 * @results: where the results of the lookup are placed
555 * @first_index: start the lookup from this key
556 * @max_items: place up to this many items at *results
557 *
558 * Performs an index-ascending scan of the tree for present items. Places
559 * them at *@results and returns the number of items which were placed at
560 * *@results.
561 *
562 * The implementation is naive.
563 */
564unsigned int
565radix_tree_gang_lookup(struct radix_tree_root *root, void **results,
566 unsigned long first_index, unsigned int max_items)
567{
568 const unsigned long max_index = radix_tree_maxindex(root->height);
569 unsigned long cur_index = first_index;
570 unsigned int ret = 0;
571
572 while (ret < max_items) {
573 unsigned int nr_found;
574 unsigned long next_index; /* Index of next search */
575
576 if (cur_index > max_index)
577 break;
578 nr_found = __lookup(root, results + ret, cur_index,
579 max_items - ret, &next_index);
580 ret += nr_found;
581 if (next_index == 0)
582 break;
583 cur_index = next_index;
584 }
585 return ret;
586}
587EXPORT_SYMBOL(radix_tree_gang_lookup);
588
589/*
590 * FIXME: the two tag_get()s here should use find_next_bit() instead of
591 * open-coding the search.
592 */
593static unsigned int
594__lookup_tag(struct radix_tree_root *root, void **results, unsigned long index,
595 unsigned int max_items, unsigned long *next_index, int tag)
596{
597 unsigned int nr_found = 0;
598 unsigned int shift;
599 unsigned int height = root->height;
600 struct radix_tree_node *slot;
601
602 shift = (height - 1) * RADIX_TREE_MAP_SHIFT;
603 slot = root->rnode;
604
605 while (height > 0) {
606 unsigned long i = (index >> shift) & RADIX_TREE_MAP_MASK;
607
608 for ( ; i < RADIX_TREE_MAP_SIZE; i++) {
609 if (tag_get(slot, tag, i)) {
610 BUG_ON(slot->slots[i] == NULL);
611 break;
612 }
613 index &= ~((1UL << shift) - 1);
614 index += 1UL << shift;
615 if (index == 0)
616 goto out; /* 32-bit wraparound */
617 }
618 if (i == RADIX_TREE_MAP_SIZE)
619 goto out;
620 height--;
621 if (height == 0) { /* Bottom level: grab some items */
622 unsigned long j = index & RADIX_TREE_MAP_MASK;
623
624 for ( ; j < RADIX_TREE_MAP_SIZE; j++) {
625 index++;
626 if (tag_get(slot, tag, j)) {
627 BUG_ON(slot->slots[j] == NULL);
628 results[nr_found++] = slot->slots[j];
629 if (nr_found == max_items)
630 goto out;
631 }
632 }
633 }
634 shift -= RADIX_TREE_MAP_SHIFT;
635 slot = slot->slots[i];
636 }
637out:
638 *next_index = index;
639 return nr_found;
640}
641
642/**
643 * radix_tree_gang_lookup_tag - perform multiple lookup on a radix tree
644 * based on a tag
645 * @root: radix tree root
646 * @results: where the results of the lookup are placed
647 * @first_index: start the lookup from this key
648 * @max_items: place up to this many items at *results
649 * @tag: the tag index
650 *
651 * Performs an index-ascending scan of the tree for present items which
652 * have the tag indexed by @tag set. Places the items at *@results and
653 * returns the number of items which were placed at *@results.
654 */
655unsigned int
656radix_tree_gang_lookup_tag(struct radix_tree_root *root, void **results,
657 unsigned long first_index, unsigned int max_items, int tag)
658{
659 const unsigned long max_index = radix_tree_maxindex(root->height);
660 unsigned long cur_index = first_index;
661 unsigned int ret = 0;
662
663 while (ret < max_items) {
664 unsigned int nr_found;
665 unsigned long next_index; /* Index of next search */
666
667 if (cur_index > max_index)
668 break;
669 nr_found = __lookup_tag(root, results + ret, cur_index,
670 max_items - ret, &next_index, tag);
671 ret += nr_found;
672 if (next_index == 0)
673 break;
674 cur_index = next_index;
675 }
676 return ret;
677}
678EXPORT_SYMBOL(radix_tree_gang_lookup_tag);
679
680/**
Nick Piggina5f51c92006-01-08 01:01:41 -0800681 * radix_tree_shrink - shrink height of a radix tree to minimal
682 * @root radix tree root
683 */
684static inline void radix_tree_shrink(struct radix_tree_root *root)
685{
686 /* try to shrink tree height */
687 while (root->height > 1 &&
688 root->rnode->count == 1 &&
689 root->rnode->slots[0]) {
690 struct radix_tree_node *to_free = root->rnode;
691
692 root->rnode = to_free->slots[0];
693 root->height--;
694 /* must only free zeroed nodes into the slab */
695 tag_clear(to_free, 0, 0);
696 tag_clear(to_free, 1, 0);
697 to_free->slots[0] = NULL;
698 to_free->count = 0;
699 radix_tree_node_free(to_free);
700 }
701}
702
703/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700704 * radix_tree_delete - delete an item from a radix tree
705 * @root: radix tree root
706 * @index: index key
707 *
708 * Remove the item at @index from the radix tree rooted at @root.
709 *
710 * Returns the address of the deleted item, or NULL if it was not present.
711 */
712void *radix_tree_delete(struct radix_tree_root *root, unsigned long index)
713{
714 struct radix_tree_path path[RADIX_TREE_MAX_PATH], *pathp = path;
715 struct radix_tree_path *orig_pathp;
Christoph Lameter201b6262005-09-06 15:16:46 -0700716 struct radix_tree_node *slot;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700717 unsigned int height, shift;
718 void *ret = NULL;
719 char tags[RADIX_TREE_TAGS];
720 int nr_cleared_tags;
Nick Piggind5274262006-01-08 01:01:41 -0800721 int tag;
722 int offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700723
724 height = root->height;
725 if (index > radix_tree_maxindex(height))
726 goto out;
727
728 shift = (height - 1) * RADIX_TREE_MAP_SHIFT;
729 pathp->node = NULL;
Christoph Lameter201b6262005-09-06 15:16:46 -0700730 slot = root->rnode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700731
Christoph Lameter201b6262005-09-06 15:16:46 -0700732 for ( ; height > 0; height--) {
Christoph Lameter201b6262005-09-06 15:16:46 -0700733 if (slot == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734 goto out;
735
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736 pathp++;
Nick Piggind5274262006-01-08 01:01:41 -0800737 offset = (index >> shift) & RADIX_TREE_MAP_MASK;
738 pathp->offset = offset;
739 pathp->node = slot;
740 slot = slot->slots[offset];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700741 shift -= RADIX_TREE_MAP_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700742 }
743
Christoph Lameter201b6262005-09-06 15:16:46 -0700744 ret = slot;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700745 if (ret == NULL)
746 goto out;
747
748 orig_pathp = pathp;
749
750 /*
751 * Clear all tags associated with the just-deleted item
752 */
Nick Piggind5274262006-01-08 01:01:41 -0800753 nr_cleared_tags = 0;
754 for (tag = 0; tag < RADIX_TREE_TAGS; tag++) {
NeilBrown90f9dd82006-02-16 14:43:01 +1100755 tags[tag] = 1;
Nick Piggind5274262006-01-08 01:01:41 -0800756 if (tag_get(pathp->node, tag, pathp->offset)) {
757 tag_clear(pathp->node, tag, pathp->offset);
NeilBrown90f9dd82006-02-16 14:43:01 +1100758 if (!any_tag_set(pathp->node, tag)) {
759 tags[tag] = 0;
760 nr_cleared_tags++;
761 }
762 }
Nick Piggind5274262006-01-08 01:01:41 -0800763 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700764
Nick Piggind5274262006-01-08 01:01:41 -0800765 for (pathp--; nr_cleared_tags && pathp->node; pathp--) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700766 for (tag = 0; tag < RADIX_TREE_TAGS; tag++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700767 if (tags[tag])
768 continue;
769
Christoph Lameter201b6262005-09-06 15:16:46 -0700770 tag_clear(pathp->node, tag, pathp->offset);
Nick Piggin6e954b92006-01-08 01:01:40 -0800771 if (any_tag_set(pathp->node, tag)) {
772 tags[tag] = 1;
773 nr_cleared_tags--;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774 }
775 }
Nick Piggind5274262006-01-08 01:01:41 -0800776 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700777
Christoph Lameter201b6262005-09-06 15:16:46 -0700778 /* Now free the nodes we do not need anymore */
779 for (pathp = orig_pathp; pathp->node; pathp--) {
780 pathp->node->slots[pathp->offset] = NULL;
Nick Piggina5f51c92006-01-08 01:01:41 -0800781 pathp->node->count--;
782
783 if (pathp->node->count) {
784 if (pathp->node == root->rnode)
785 radix_tree_shrink(root);
Christoph Lameter201b6262005-09-06 15:16:46 -0700786 goto out;
Nick Piggina5f51c92006-01-08 01:01:41 -0800787 }
Christoph Lameter201b6262005-09-06 15:16:46 -0700788
789 /* Node with zero slots in use so free it */
790 radix_tree_node_free(pathp->node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700791 }
Christoph Lameter201b6262005-09-06 15:16:46 -0700792 root->rnode = NULL;
793 root->height = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700794out:
795 return ret;
796}
797EXPORT_SYMBOL(radix_tree_delete);
798
799/**
800 * radix_tree_tagged - test whether any items in the tree are tagged
801 * @root: radix tree root
802 * @tag: tag to test
803 */
804int radix_tree_tagged(struct radix_tree_root *root, int tag)
805{
Nick Piggin6e954b92006-01-08 01:01:40 -0800806 struct radix_tree_node *rnode;
807 rnode = root->rnode;
808 if (!rnode)
809 return 0;
810 return any_tag_set(rnode, tag);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700811}
812EXPORT_SYMBOL(radix_tree_tagged);
813
814static void
815radix_tree_node_ctor(void *node, kmem_cache_t *cachep, unsigned long flags)
816{
817 memset(node, 0, sizeof(struct radix_tree_node));
818}
819
820static __init unsigned long __maxindex(unsigned int height)
821{
822 unsigned int tmp = height * RADIX_TREE_MAP_SHIFT;
823 unsigned long index = (~0UL >> (RADIX_TREE_INDEX_BITS - tmp - 1)) >> 1;
824
825 if (tmp >= RADIX_TREE_INDEX_BITS)
826 index = ~0UL;
827 return index;
828}
829
830static __init void radix_tree_init_maxindex(void)
831{
832 unsigned int i;
833
834 for (i = 0; i < ARRAY_SIZE(height_to_maxindex); i++)
835 height_to_maxindex[i] = __maxindex(i);
836}
837
838#ifdef CONFIG_HOTPLUG_CPU
839static int radix_tree_callback(struct notifier_block *nfb,
840 unsigned long action,
841 void *hcpu)
842{
843 int cpu = (long)hcpu;
844 struct radix_tree_preload *rtp;
845
846 /* Free per-cpu pool of perloaded nodes */
847 if (action == CPU_DEAD) {
848 rtp = &per_cpu(radix_tree_preloads, cpu);
849 while (rtp->nr) {
850 kmem_cache_free(radix_tree_node_cachep,
851 rtp->nodes[rtp->nr-1]);
852 rtp->nodes[rtp->nr-1] = NULL;
853 rtp->nr--;
854 }
855 }
856 return NOTIFY_OK;
857}
858#endif /* CONFIG_HOTPLUG_CPU */
859
860void __init radix_tree_init(void)
861{
862 radix_tree_node_cachep = kmem_cache_create("radix_tree_node",
863 sizeof(struct radix_tree_node), 0,
864 SLAB_PANIC, radix_tree_node_ctor, NULL);
865 radix_tree_init_maxindex();
866 hotcpu_notifier(radix_tree_callback, 0);
867}