blob: d69ddbe438655be6b65672d2c7366371baadfd85 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2001 Momchil Velikov
3 * Portions Copyright (C) 2001 Christoph Hellwig
Christoph Lameter201b6262005-09-06 15:16:46 -07004 * Copyright (C) 2005 SGI, Christoph Lameter <clameter@sgi.com>
Nick Piggin7cf9c2c2006-12-06 20:33:44 -08005 * Copyright (C) 2006 Nick Piggin
Linus Torvalds1da177e2005-04-16 15:20:36 -07006 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License as
9 * published by the Free Software Foundation; either version 2, or (at
10 * your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
20 */
21
22#include <linux/errno.h>
23#include <linux/init.h>
24#include <linux/kernel.h>
25#include <linux/module.h>
26#include <linux/radix-tree.h>
27#include <linux/percpu.h>
28#include <linux/slab.h>
29#include <linux/notifier.h>
30#include <linux/cpu.h>
31#include <linux/gfp.h>
32#include <linux/string.h>
33#include <linux/bitops.h>
Nick Piggin7cf9c2c2006-12-06 20:33:44 -080034#include <linux/rcupdate.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070035
36
37#ifdef __KERNEL__
Nick Piggincfd9b7d2006-06-23 02:03:22 -070038#define RADIX_TREE_MAP_SHIFT (CONFIG_BASE_SMALL ? 4 : 6)
Linus Torvalds1da177e2005-04-16 15:20:36 -070039#else
40#define RADIX_TREE_MAP_SHIFT 3 /* For more stressful testing */
41#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070042
43#define RADIX_TREE_MAP_SIZE (1UL << RADIX_TREE_MAP_SHIFT)
44#define RADIX_TREE_MAP_MASK (RADIX_TREE_MAP_SIZE-1)
45
46#define RADIX_TREE_TAG_LONGS \
47 ((RADIX_TREE_MAP_SIZE + BITS_PER_LONG - 1) / BITS_PER_LONG)
48
49struct radix_tree_node {
Nick Piggin7cf9c2c2006-12-06 20:33:44 -080050 unsigned int height; /* Height from the bottom */
Linus Torvalds1da177e2005-04-16 15:20:36 -070051 unsigned int count;
Nick Piggin7cf9c2c2006-12-06 20:33:44 -080052 struct rcu_head rcu_head;
Linus Torvalds1da177e2005-04-16 15:20:36 -070053 void *slots[RADIX_TREE_MAP_SIZE];
Jonathan Corbetdaff89f2006-03-25 03:08:05 -080054 unsigned long tags[RADIX_TREE_MAX_TAGS][RADIX_TREE_TAG_LONGS];
Linus Torvalds1da177e2005-04-16 15:20:36 -070055};
56
57struct radix_tree_path {
Christoph Lameter201b6262005-09-06 15:16:46 -070058 struct radix_tree_node *node;
Linus Torvalds1da177e2005-04-16 15:20:36 -070059 int offset;
60};
61
62#define RADIX_TREE_INDEX_BITS (8 /* CHAR_BIT */ * sizeof(unsigned long))
63#define RADIX_TREE_MAX_PATH (RADIX_TREE_INDEX_BITS/RADIX_TREE_MAP_SHIFT + 2)
64
Christoph Lameter6c036522005-07-07 17:56:59 -070065static unsigned long height_to_maxindex[RADIX_TREE_MAX_PATH] __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -070066
67/*
68 * Radix tree node cache.
69 */
Christoph Lametere18b8902006-12-06 20:33:20 -080070static struct kmem_cache *radix_tree_node_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -070071
72/*
73 * Per-cpu pool of preloaded nodes
74 */
75struct radix_tree_preload {
76 int nr;
77 struct radix_tree_node *nodes[RADIX_TREE_MAX_PATH];
78};
79DEFINE_PER_CPU(struct radix_tree_preload, radix_tree_preloads) = { 0, };
80
Nick Piggin612d6c12006-06-23 02:03:22 -070081static inline gfp_t root_gfp_mask(struct radix_tree_root *root)
82{
83 return root->gfp_mask & __GFP_BITS_MASK;
84}
85
Linus Torvalds1da177e2005-04-16 15:20:36 -070086/*
87 * This assumes that the caller has performed appropriate preallocation, and
88 * that the caller has pinned this thread of control to the current CPU.
89 */
90static struct radix_tree_node *
91radix_tree_node_alloc(struct radix_tree_root *root)
92{
93 struct radix_tree_node *ret;
Nick Piggin612d6c12006-06-23 02:03:22 -070094 gfp_t gfp_mask = root_gfp_mask(root);
Linus Torvalds1da177e2005-04-16 15:20:36 -070095
Nick Piggin612d6c12006-06-23 02:03:22 -070096 ret = kmem_cache_alloc(radix_tree_node_cachep, gfp_mask);
97 if (ret == NULL && !(gfp_mask & __GFP_WAIT)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -070098 struct radix_tree_preload *rtp;
99
100 rtp = &__get_cpu_var(radix_tree_preloads);
101 if (rtp->nr) {
102 ret = rtp->nodes[rtp->nr - 1];
103 rtp->nodes[rtp->nr - 1] = NULL;
104 rtp->nr--;
105 }
106 }
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800107 BUG_ON(radix_tree_is_direct_ptr(ret));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108 return ret;
109}
110
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800111static void radix_tree_node_rcu_free(struct rcu_head *head)
112{
113 struct radix_tree_node *node =
114 container_of(head, struct radix_tree_node, rcu_head);
115 kmem_cache_free(radix_tree_node_cachep, node);
116}
117
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118static inline void
119radix_tree_node_free(struct radix_tree_node *node)
120{
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800121 call_rcu(&node->rcu_head, radix_tree_node_rcu_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122}
123
124/*
125 * Load up this CPU's radix_tree_node buffer with sufficient objects to
126 * ensure that the addition of a single element in the tree cannot fail. On
127 * success, return zero, with preemption disabled. On error, return -ENOMEM
128 * with preemption not disabled.
129 */
Al Virodd0fc662005-10-07 07:46:04 +0100130int radix_tree_preload(gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131{
132 struct radix_tree_preload *rtp;
133 struct radix_tree_node *node;
134 int ret = -ENOMEM;
135
136 preempt_disable();
137 rtp = &__get_cpu_var(radix_tree_preloads);
138 while (rtp->nr < ARRAY_SIZE(rtp->nodes)) {
139 preempt_enable();
140 node = kmem_cache_alloc(radix_tree_node_cachep, gfp_mask);
141 if (node == NULL)
142 goto out;
143 preempt_disable();
144 rtp = &__get_cpu_var(radix_tree_preloads);
145 if (rtp->nr < ARRAY_SIZE(rtp->nodes))
146 rtp->nodes[rtp->nr++] = node;
147 else
148 kmem_cache_free(radix_tree_node_cachep, node);
149 }
150 ret = 0;
151out:
152 return ret;
153}
154
Jonathan Corbetdaff89f2006-03-25 03:08:05 -0800155static inline void tag_set(struct radix_tree_node *node, unsigned int tag,
156 int offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157{
Nick Piggind5274262006-01-08 01:01:41 -0800158 __set_bit(offset, node->tags[tag]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159}
160
Jonathan Corbetdaff89f2006-03-25 03:08:05 -0800161static inline void tag_clear(struct radix_tree_node *node, unsigned int tag,
162 int offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163{
Nick Piggind5274262006-01-08 01:01:41 -0800164 __clear_bit(offset, node->tags[tag]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165}
166
Jonathan Corbetdaff89f2006-03-25 03:08:05 -0800167static inline int tag_get(struct radix_tree_node *node, unsigned int tag,
168 int offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169{
Nick Piggind5274262006-01-08 01:01:41 -0800170 return test_bit(offset, node->tags[tag]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700171}
172
Nick Piggin612d6c12006-06-23 02:03:22 -0700173static inline void root_tag_set(struct radix_tree_root *root, unsigned int tag)
174{
Al Viro20241ad2006-10-10 22:47:57 +0100175 root->gfp_mask |= (__force gfp_t)(1 << (tag + __GFP_BITS_SHIFT));
Nick Piggin612d6c12006-06-23 02:03:22 -0700176}
177
178
179static inline void root_tag_clear(struct radix_tree_root *root, unsigned int tag)
180{
Al Viro20241ad2006-10-10 22:47:57 +0100181 root->gfp_mask &= (__force gfp_t)~(1 << (tag + __GFP_BITS_SHIFT));
Nick Piggin612d6c12006-06-23 02:03:22 -0700182}
183
184static inline void root_tag_clear_all(struct radix_tree_root *root)
185{
186 root->gfp_mask &= __GFP_BITS_MASK;
187}
188
189static inline int root_tag_get(struct radix_tree_root *root, unsigned int tag)
190{
Al Viro20241ad2006-10-10 22:47:57 +0100191 return (__force unsigned)root->gfp_mask & (1 << (tag + __GFP_BITS_SHIFT));
Nick Piggin612d6c12006-06-23 02:03:22 -0700192}
193
Linus Torvalds1da177e2005-04-16 15:20:36 -0700194/*
Nick Piggin6e954b92006-01-08 01:01:40 -0800195 * Returns 1 if any slot in the node has this tag set.
196 * Otherwise returns 0.
197 */
Jonathan Corbetdaff89f2006-03-25 03:08:05 -0800198static inline int any_tag_set(struct radix_tree_node *node, unsigned int tag)
Nick Piggin6e954b92006-01-08 01:01:40 -0800199{
200 int idx;
201 for (idx = 0; idx < RADIX_TREE_TAG_LONGS; idx++) {
202 if (node->tags[tag][idx])
203 return 1;
204 }
205 return 0;
206}
207
208/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700209 * Return the maximum key which can be store into a
210 * radix tree with height HEIGHT.
211 */
212static inline unsigned long radix_tree_maxindex(unsigned int height)
213{
214 return height_to_maxindex[height];
215}
216
217/*
218 * Extend a radix tree so it can store key @index.
219 */
220static int radix_tree_extend(struct radix_tree_root *root, unsigned long index)
221{
222 struct radix_tree_node *node;
223 unsigned int height;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224 int tag;
225
226 /* Figure out what the height should be. */
227 height = root->height + 1;
228 while (index > radix_tree_maxindex(height))
229 height++;
230
231 if (root->rnode == NULL) {
232 root->height = height;
233 goto out;
234 }
235
Linus Torvalds1da177e2005-04-16 15:20:36 -0700236 do {
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800237 unsigned int newheight;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238 if (!(node = radix_tree_node_alloc(root)))
239 return -ENOMEM;
240
241 /* Increase the height. */
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800242 node->slots[0] = radix_tree_direct_to_ptr(root->rnode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243
244 /* Propagate the aggregated tag info into the new root */
Jonathan Corbetdaff89f2006-03-25 03:08:05 -0800245 for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++) {
Nick Piggin612d6c12006-06-23 02:03:22 -0700246 if (root_tag_get(root, tag))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247 tag_set(node, tag, 0);
248 }
249
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800250 newheight = root->height+1;
251 node->height = newheight;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700252 node->count = 1;
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800253 rcu_assign_pointer(root->rnode, node);
254 root->height = newheight;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255 } while (height > root->height);
256out:
257 return 0;
258}
259
260/**
261 * radix_tree_insert - insert into a radix tree
262 * @root: radix tree root
263 * @index: index key
264 * @item: item to insert
265 *
266 * Insert an item into the radix tree at position @index.
267 */
268int radix_tree_insert(struct radix_tree_root *root,
269 unsigned long index, void *item)
270{
Christoph Lameter201b6262005-09-06 15:16:46 -0700271 struct radix_tree_node *node = NULL, *slot;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272 unsigned int height, shift;
273 int offset;
274 int error;
275
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800276 BUG_ON(radix_tree_is_direct_ptr(item));
277
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278 /* Make sure the tree is high enough. */
Nick Piggin612d6c12006-06-23 02:03:22 -0700279 if (index > radix_tree_maxindex(root->height)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700280 error = radix_tree_extend(root, index);
281 if (error)
282 return error;
283 }
284
Christoph Lameter201b6262005-09-06 15:16:46 -0700285 slot = root->rnode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700286 height = root->height;
287 shift = (height-1) * RADIX_TREE_MAP_SHIFT;
288
289 offset = 0; /* uninitialised var warning */
Nick Piggin612d6c12006-06-23 02:03:22 -0700290 while (height > 0) {
Christoph Lameter201b6262005-09-06 15:16:46 -0700291 if (slot == NULL) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292 /* Have to add a child node. */
Christoph Lameter201b6262005-09-06 15:16:46 -0700293 if (!(slot = radix_tree_node_alloc(root)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700294 return -ENOMEM;
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800295 slot->height = height;
Christoph Lameter201b6262005-09-06 15:16:46 -0700296 if (node) {
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800297 rcu_assign_pointer(node->slots[offset], slot);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700298 node->count++;
Christoph Lameter201b6262005-09-06 15:16:46 -0700299 } else
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800300 rcu_assign_pointer(root->rnode, slot);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301 }
302
303 /* Go a level down */
304 offset = (index >> shift) & RADIX_TREE_MAP_MASK;
Christoph Lameter201b6262005-09-06 15:16:46 -0700305 node = slot;
306 slot = node->slots[offset];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700307 shift -= RADIX_TREE_MAP_SHIFT;
308 height--;
Nick Piggin612d6c12006-06-23 02:03:22 -0700309 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310
Christoph Lameter201b6262005-09-06 15:16:46 -0700311 if (slot != NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312 return -EEXIST;
Christoph Lameter201b6262005-09-06 15:16:46 -0700313
Nick Piggin612d6c12006-06-23 02:03:22 -0700314 if (node) {
315 node->count++;
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800316 rcu_assign_pointer(node->slots[offset], item);
Nick Piggin612d6c12006-06-23 02:03:22 -0700317 BUG_ON(tag_get(node, 0, offset));
318 BUG_ON(tag_get(node, 1, offset));
319 } else {
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800320 rcu_assign_pointer(root->rnode, radix_tree_ptr_to_direct(item));
Nick Piggin612d6c12006-06-23 02:03:22 -0700321 BUG_ON(root_tag_get(root, 0));
322 BUG_ON(root_tag_get(root, 1));
323 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700324
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325 return 0;
326}
327EXPORT_SYMBOL(radix_tree_insert);
328
Hans Reisera4331362005-11-07 00:59:29 -0800329/**
330 * radix_tree_lookup_slot - lookup a slot in a radix tree
331 * @root: radix tree root
332 * @index: index key
333 *
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800334 * Returns: the slot corresponding to the position @index in the
335 * radix tree @root. This is useful for update-if-exists operations.
336 *
337 * This function cannot be called under rcu_read_lock, it must be
338 * excluded from writers, as must the returned slot for subsequent
339 * use by radix_tree_deref_slot() and radix_tree_replace slot.
340 * Caller must hold tree write locked across slot lookup and
341 * replace.
Hans Reisera4331362005-11-07 00:59:29 -0800342 */
343void **radix_tree_lookup_slot(struct radix_tree_root *root, unsigned long index)
344{
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800345 unsigned int height, shift;
346 struct radix_tree_node *node, **slot;
347
348 node = root->rnode;
349 if (node == NULL)
350 return NULL;
351
352 if (radix_tree_is_direct_ptr(node)) {
353 if (index > 0)
354 return NULL;
355 return (void **)&root->rnode;
356 }
357
358 height = node->height;
359 if (index > radix_tree_maxindex(height))
360 return NULL;
361
362 shift = (height-1) * RADIX_TREE_MAP_SHIFT;
363
364 do {
365 slot = (struct radix_tree_node **)
366 (node->slots + ((index>>shift) & RADIX_TREE_MAP_MASK));
367 node = *slot;
368 if (node == NULL)
369 return NULL;
370
371 shift -= RADIX_TREE_MAP_SHIFT;
372 height--;
373 } while (height > 0);
374
375 return (void **)slot;
Hans Reisera4331362005-11-07 00:59:29 -0800376}
377EXPORT_SYMBOL(radix_tree_lookup_slot);
378
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379/**
380 * radix_tree_lookup - perform lookup operation on a radix tree
381 * @root: radix tree root
382 * @index: index key
383 *
384 * Lookup the item at the position @index in the radix tree @root.
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800385 *
386 * This function can be called under rcu_read_lock, however the caller
387 * must manage lifetimes of leaf nodes (eg. RCU may also be used to free
388 * them safely). No RCU barriers are required to access or modify the
389 * returned item, however.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390 */
391void *radix_tree_lookup(struct radix_tree_root *root, unsigned long index)
392{
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800393 unsigned int height, shift;
394 struct radix_tree_node *node, **slot;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700395
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800396 node = rcu_dereference(root->rnode);
397 if (node == NULL)
398 return NULL;
399
400 if (radix_tree_is_direct_ptr(node)) {
401 if (index > 0)
402 return NULL;
403 return radix_tree_direct_to_ptr(node);
404 }
405
406 height = node->height;
407 if (index > radix_tree_maxindex(height))
408 return NULL;
409
410 shift = (height-1) * RADIX_TREE_MAP_SHIFT;
411
412 do {
413 slot = (struct radix_tree_node **)
414 (node->slots + ((index>>shift) & RADIX_TREE_MAP_MASK));
415 node = rcu_dereference(*slot);
416 if (node == NULL)
417 return NULL;
418
419 shift -= RADIX_TREE_MAP_SHIFT;
420 height--;
421 } while (height > 0);
422
423 return node;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424}
425EXPORT_SYMBOL(radix_tree_lookup);
426
427/**
428 * radix_tree_tag_set - set a tag on a radix tree node
429 * @root: radix tree root
430 * @index: index key
431 * @tag: tag index
432 *
Jonathan Corbetdaff89f2006-03-25 03:08:05 -0800433 * Set the search tag (which must be < RADIX_TREE_MAX_TAGS)
434 * corresponding to @index in the radix tree. From
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435 * the root all the way down to the leaf node.
436 *
437 * Returns the address of the tagged item. Setting a tag on a not-present
438 * item is a bug.
439 */
440void *radix_tree_tag_set(struct radix_tree_root *root,
Jonathan Corbetdaff89f2006-03-25 03:08:05 -0800441 unsigned long index, unsigned int tag)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700442{
443 unsigned int height, shift;
Christoph Lameter201b6262005-09-06 15:16:46 -0700444 struct radix_tree_node *slot;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445
446 height = root->height;
Peter Zijlstra4c91c362006-06-23 02:03:25 -0700447 BUG_ON(index > radix_tree_maxindex(height));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700448
Christoph Lameter201b6262005-09-06 15:16:46 -0700449 slot = root->rnode;
Nick Piggin612d6c12006-06-23 02:03:22 -0700450 shift = (height - 1) * RADIX_TREE_MAP_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700451
452 while (height > 0) {
453 int offset;
454
455 offset = (index >> shift) & RADIX_TREE_MAP_MASK;
Nick Piggind5274262006-01-08 01:01:41 -0800456 if (!tag_get(slot, tag, offset))
457 tag_set(slot, tag, offset);
Christoph Lameter201b6262005-09-06 15:16:46 -0700458 slot = slot->slots[offset];
459 BUG_ON(slot == NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700460 shift -= RADIX_TREE_MAP_SHIFT;
461 height--;
462 }
463
Nick Piggin612d6c12006-06-23 02:03:22 -0700464 /* set the root's tag bit */
465 if (slot && !root_tag_get(root, tag))
466 root_tag_set(root, tag);
467
Christoph Lameter201b6262005-09-06 15:16:46 -0700468 return slot;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469}
470EXPORT_SYMBOL(radix_tree_tag_set);
471
472/**
473 * radix_tree_tag_clear - clear a tag on a radix tree node
474 * @root: radix tree root
475 * @index: index key
476 * @tag: tag index
477 *
Jonathan Corbetdaff89f2006-03-25 03:08:05 -0800478 * Clear the search tag (which must be < RADIX_TREE_MAX_TAGS)
479 * corresponding to @index in the radix tree. If
Linus Torvalds1da177e2005-04-16 15:20:36 -0700480 * this causes the leaf node to have no tags set then clear the tag in the
481 * next-to-leaf node, etc.
482 *
483 * Returns the address of the tagged item on success, else NULL. ie:
484 * has the same return value and semantics as radix_tree_lookup().
485 */
486void *radix_tree_tag_clear(struct radix_tree_root *root,
Jonathan Corbetdaff89f2006-03-25 03:08:05 -0800487 unsigned long index, unsigned int tag)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488{
489 struct radix_tree_path path[RADIX_TREE_MAX_PATH], *pathp = path;
Nick Piggin612d6c12006-06-23 02:03:22 -0700490 struct radix_tree_node *slot = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700491 unsigned int height, shift;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492
493 height = root->height;
494 if (index > radix_tree_maxindex(height))
495 goto out;
496
497 shift = (height - 1) * RADIX_TREE_MAP_SHIFT;
498 pathp->node = NULL;
Christoph Lameter201b6262005-09-06 15:16:46 -0700499 slot = root->rnode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500
501 while (height > 0) {
502 int offset;
503
Christoph Lameter201b6262005-09-06 15:16:46 -0700504 if (slot == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700505 goto out;
506
507 offset = (index >> shift) & RADIX_TREE_MAP_MASK;
508 pathp[1].offset = offset;
Christoph Lameter201b6262005-09-06 15:16:46 -0700509 pathp[1].node = slot;
510 slot = slot->slots[offset];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700511 pathp++;
512 shift -= RADIX_TREE_MAP_SHIFT;
513 height--;
514 }
515
Nick Piggin612d6c12006-06-23 02:03:22 -0700516 if (slot == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700517 goto out;
518
Nick Piggin612d6c12006-06-23 02:03:22 -0700519 while (pathp->node) {
Nick Piggind5274262006-01-08 01:01:41 -0800520 if (!tag_get(pathp->node, tag, pathp->offset))
521 goto out;
Christoph Lameter201b6262005-09-06 15:16:46 -0700522 tag_clear(pathp->node, tag, pathp->offset);
Nick Piggin6e954b92006-01-08 01:01:40 -0800523 if (any_tag_set(pathp->node, tag))
524 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525 pathp--;
Nick Piggin612d6c12006-06-23 02:03:22 -0700526 }
527
528 /* clear the root's tag bit */
529 if (root_tag_get(root, tag))
530 root_tag_clear(root, tag);
531
Linus Torvalds1da177e2005-04-16 15:20:36 -0700532out:
Nick Piggin612d6c12006-06-23 02:03:22 -0700533 return slot;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700534}
535EXPORT_SYMBOL(radix_tree_tag_clear);
536
537#ifndef __KERNEL__ /* Only the test harness uses this at present */
538/**
Marcelo Tosatti32605a12005-09-06 15:16:48 -0700539 * radix_tree_tag_get - get a tag on a radix tree node
540 * @root: radix tree root
541 * @index: index key
Jonathan Corbetdaff89f2006-03-25 03:08:05 -0800542 * @tag: tag index (< RADIX_TREE_MAX_TAGS)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543 *
Marcelo Tosatti32605a12005-09-06 15:16:48 -0700544 * Return values:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545 *
Nick Piggin612d6c12006-06-23 02:03:22 -0700546 * 0: tag not present or not set
547 * 1: tag set
Linus Torvalds1da177e2005-04-16 15:20:36 -0700548 */
549int radix_tree_tag_get(struct radix_tree_root *root,
Jonathan Corbetdaff89f2006-03-25 03:08:05 -0800550 unsigned long index, unsigned int tag)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700551{
552 unsigned int height, shift;
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800553 struct radix_tree_node *node;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554 int saw_unset_tag = 0;
555
Nick Piggin612d6c12006-06-23 02:03:22 -0700556 /* check the root's tag bit */
557 if (!root_tag_get(root, tag))
558 return 0;
559
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800560 node = rcu_dereference(root->rnode);
561 if (node == NULL)
562 return 0;
563
564 if (radix_tree_is_direct_ptr(node))
565 return (index == 0);
566
567 height = node->height;
568 if (index > radix_tree_maxindex(height))
569 return 0;
Nick Piggin612d6c12006-06-23 02:03:22 -0700570
Linus Torvalds1da177e2005-04-16 15:20:36 -0700571 shift = (height - 1) * RADIX_TREE_MAP_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572
573 for ( ; ; ) {
574 int offset;
575
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800576 if (node == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700577 return 0;
578
579 offset = (index >> shift) & RADIX_TREE_MAP_MASK;
580
581 /*
582 * This is just a debug check. Later, we can bale as soon as
583 * we see an unset tag.
584 */
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800585 if (!tag_get(node, tag, offset))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586 saw_unset_tag = 1;
587 if (height == 1) {
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800588 int ret = tag_get(node, tag, offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589
590 BUG_ON(ret && saw_unset_tag);
Wu Fengguange5dcd902006-06-25 05:48:14 -0700591 return !!ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592 }
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800593 node = rcu_dereference(node->slots[offset]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594 shift -= RADIX_TREE_MAP_SHIFT;
595 height--;
596 }
597}
598EXPORT_SYMBOL(radix_tree_tag_get);
599#endif
600
601static unsigned int
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800602__lookup(struct radix_tree_node *slot, void **results, unsigned long index,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700603 unsigned int max_items, unsigned long *next_index)
604{
605 unsigned int nr_found = 0;
Christoph Lameter201b6262005-09-06 15:16:46 -0700606 unsigned int shift, height;
Christoph Lameter201b6262005-09-06 15:16:46 -0700607 unsigned long i;
608
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800609 height = slot->height;
610 if (height == 0)
Christoph Lameter201b6262005-09-06 15:16:46 -0700611 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700612 shift = (height-1) * RADIX_TREE_MAP_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613
Christoph Lameter201b6262005-09-06 15:16:46 -0700614 for ( ; height > 1; height--) {
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800615 i = (index >> shift) & RADIX_TREE_MAP_MASK;
616 for (;;) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617 if (slot->slots[i] != NULL)
618 break;
619 index &= ~((1UL << shift) - 1);
620 index += 1UL << shift;
621 if (index == 0)
622 goto out; /* 32-bit wraparound */
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800623 i++;
624 if (i == RADIX_TREE_MAP_SIZE)
625 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700626 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627
Linus Torvalds1da177e2005-04-16 15:20:36 -0700628 shift -= RADIX_TREE_MAP_SHIFT;
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800629 slot = rcu_dereference(slot->slots[i]);
630 if (slot == NULL)
631 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700632 }
Christoph Lameter201b6262005-09-06 15:16:46 -0700633
634 /* Bottom level: grab some items */
635 for (i = index & RADIX_TREE_MAP_MASK; i < RADIX_TREE_MAP_SIZE; i++) {
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800636 struct radix_tree_node *node;
Christoph Lameter201b6262005-09-06 15:16:46 -0700637 index++;
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800638 node = slot->slots[i];
639 if (node) {
640 results[nr_found++] = rcu_dereference(node);
Christoph Lameter201b6262005-09-06 15:16:46 -0700641 if (nr_found == max_items)
642 goto out;
643 }
644 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700645out:
646 *next_index = index;
647 return nr_found;
648}
649
650/**
651 * radix_tree_gang_lookup - perform multiple lookup on a radix tree
652 * @root: radix tree root
653 * @results: where the results of the lookup are placed
654 * @first_index: start the lookup from this key
655 * @max_items: place up to this many items at *results
656 *
657 * Performs an index-ascending scan of the tree for present items. Places
658 * them at *@results and returns the number of items which were placed at
659 * *@results.
660 *
661 * The implementation is naive.
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800662 *
663 * Like radix_tree_lookup, radix_tree_gang_lookup may be called under
664 * rcu_read_lock. In this case, rather than the returned results being
665 * an atomic snapshot of the tree at a single point in time, the semantics
666 * of an RCU protected gang lookup are as though multiple radix_tree_lookups
667 * have been issued in individual locks, and results stored in 'results'.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668 */
669unsigned int
670radix_tree_gang_lookup(struct radix_tree_root *root, void **results,
671 unsigned long first_index, unsigned int max_items)
672{
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800673 unsigned long max_index;
674 struct radix_tree_node *node;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700675 unsigned long cur_index = first_index;
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800676 unsigned int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800678 node = rcu_dereference(root->rnode);
679 if (!node)
680 return 0;
681
682 if (radix_tree_is_direct_ptr(node)) {
683 if (first_index > 0)
684 return 0;
685 node = radix_tree_direct_to_ptr(node);
686 results[0] = rcu_dereference(node);
687 return 1;
688 }
689
690 max_index = radix_tree_maxindex(node->height);
691
692 ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700693 while (ret < max_items) {
694 unsigned int nr_found;
695 unsigned long next_index; /* Index of next search */
696
697 if (cur_index > max_index)
698 break;
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800699 nr_found = __lookup(node, results + ret, cur_index,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700700 max_items - ret, &next_index);
701 ret += nr_found;
702 if (next_index == 0)
703 break;
704 cur_index = next_index;
705 }
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800706
Linus Torvalds1da177e2005-04-16 15:20:36 -0700707 return ret;
708}
709EXPORT_SYMBOL(radix_tree_gang_lookup);
710
711/*
712 * FIXME: the two tag_get()s here should use find_next_bit() instead of
713 * open-coding the search.
714 */
715static unsigned int
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800716__lookup_tag(struct radix_tree_node *slot, void **results, unsigned long index,
Jonathan Corbetdaff89f2006-03-25 03:08:05 -0800717 unsigned int max_items, unsigned long *next_index, unsigned int tag)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700718{
719 unsigned int nr_found = 0;
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800720 unsigned int shift, height;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700721
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800722 height = slot->height;
723 if (height == 0)
Nick Piggin612d6c12006-06-23 02:03:22 -0700724 goto out;
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800725 shift = (height-1) * RADIX_TREE_MAP_SHIFT;
Nick Piggin612d6c12006-06-23 02:03:22 -0700726
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800727 while (height > 0) {
728 unsigned long i = (index >> shift) & RADIX_TREE_MAP_MASK ;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700729
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800730 for (;;) {
731 if (tag_get(slot, tag, i))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700732 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700733 index &= ~((1UL << shift) - 1);
734 index += 1UL << shift;
735 if (index == 0)
736 goto out; /* 32-bit wraparound */
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800737 i++;
738 if (i == RADIX_TREE_MAP_SIZE)
739 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700740 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700741 height--;
742 if (height == 0) { /* Bottom level: grab some items */
743 unsigned long j = index & RADIX_TREE_MAP_MASK;
744
745 for ( ; j < RADIX_TREE_MAP_SIZE; j++) {
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800746 struct radix_tree_node *node;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700747 index++;
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800748 if (!tag_get(slot, tag, j))
749 continue;
750 node = slot->slots[j];
751 /*
752 * Even though the tag was found set, we need to
753 * recheck that we have a non-NULL node, because
754 * if this lookup is lockless, it may have been
755 * subsequently deleted.
756 *
757 * Similar care must be taken in any place that
758 * lookup ->slots[x] without a lock (ie. can't
759 * rely on its value remaining the same).
760 */
761 if (node) {
762 node = rcu_dereference(node);
763 results[nr_found++] = node;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700764 if (nr_found == max_items)
765 goto out;
766 }
767 }
768 }
769 shift -= RADIX_TREE_MAP_SHIFT;
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800770 slot = rcu_dereference(slot->slots[i]);
771 if (slot == NULL)
772 break;
773 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774out:
775 *next_index = index;
776 return nr_found;
777}
778
779/**
780 * radix_tree_gang_lookup_tag - perform multiple lookup on a radix tree
781 * based on a tag
782 * @root: radix tree root
783 * @results: where the results of the lookup are placed
784 * @first_index: start the lookup from this key
785 * @max_items: place up to this many items at *results
Jonathan Corbetdaff89f2006-03-25 03:08:05 -0800786 * @tag: the tag index (< RADIX_TREE_MAX_TAGS)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700787 *
788 * Performs an index-ascending scan of the tree for present items which
789 * have the tag indexed by @tag set. Places the items at *@results and
790 * returns the number of items which were placed at *@results.
791 */
792unsigned int
793radix_tree_gang_lookup_tag(struct radix_tree_root *root, void **results,
Jonathan Corbetdaff89f2006-03-25 03:08:05 -0800794 unsigned long first_index, unsigned int max_items,
795 unsigned int tag)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700796{
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800797 struct radix_tree_node *node;
798 unsigned long max_index;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799 unsigned long cur_index = first_index;
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800800 unsigned int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700801
Nick Piggin612d6c12006-06-23 02:03:22 -0700802 /* check the root's tag bit */
803 if (!root_tag_get(root, tag))
804 return 0;
805
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800806 node = rcu_dereference(root->rnode);
807 if (!node)
808 return 0;
809
810 if (radix_tree_is_direct_ptr(node)) {
811 if (first_index > 0)
812 return 0;
813 node = radix_tree_direct_to_ptr(node);
814 results[0] = rcu_dereference(node);
815 return 1;
816 }
817
818 max_index = radix_tree_maxindex(node->height);
819
820 ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821 while (ret < max_items) {
822 unsigned int nr_found;
823 unsigned long next_index; /* Index of next search */
824
825 if (cur_index > max_index)
826 break;
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800827 nr_found = __lookup_tag(node, results + ret, cur_index,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700828 max_items - ret, &next_index, tag);
829 ret += nr_found;
830 if (next_index == 0)
831 break;
832 cur_index = next_index;
833 }
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800834
Linus Torvalds1da177e2005-04-16 15:20:36 -0700835 return ret;
836}
837EXPORT_SYMBOL(radix_tree_gang_lookup_tag);
838
839/**
Nick Piggina5f51c92006-01-08 01:01:41 -0800840 * radix_tree_shrink - shrink height of a radix tree to minimal
841 * @root radix tree root
842 */
843static inline void radix_tree_shrink(struct radix_tree_root *root)
844{
845 /* try to shrink tree height */
Nick Piggin612d6c12006-06-23 02:03:22 -0700846 while (root->height > 0 &&
Nick Piggina5f51c92006-01-08 01:01:41 -0800847 root->rnode->count == 1 &&
848 root->rnode->slots[0]) {
849 struct radix_tree_node *to_free = root->rnode;
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800850 void *newptr;
Nick Piggina5f51c92006-01-08 01:01:41 -0800851
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800852 /*
853 * We don't need rcu_assign_pointer(), since we are simply
854 * moving the node from one part of the tree to another. If
855 * it was safe to dereference the old pointer to it
856 * (to_free->slots[0]), it will be safe to dereference the new
857 * one (root->rnode).
858 */
859 newptr = to_free->slots[0];
860 if (root->height == 1)
861 newptr = radix_tree_ptr_to_direct(newptr);
862 root->rnode = newptr;
Nick Piggina5f51c92006-01-08 01:01:41 -0800863 root->height--;
864 /* must only free zeroed nodes into the slab */
865 tag_clear(to_free, 0, 0);
866 tag_clear(to_free, 1, 0);
867 to_free->slots[0] = NULL;
868 to_free->count = 0;
869 radix_tree_node_free(to_free);
870 }
871}
872
873/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700874 * radix_tree_delete - delete an item from a radix tree
875 * @root: radix tree root
876 * @index: index key
877 *
878 * Remove the item at @index from the radix tree rooted at @root.
879 *
880 * Returns the address of the deleted item, or NULL if it was not present.
881 */
882void *radix_tree_delete(struct radix_tree_root *root, unsigned long index)
883{
884 struct radix_tree_path path[RADIX_TREE_MAX_PATH], *pathp = path;
Nick Piggin612d6c12006-06-23 02:03:22 -0700885 struct radix_tree_node *slot = NULL;
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800886 struct radix_tree_node *to_free;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700887 unsigned int height, shift;
Nick Piggind5274262006-01-08 01:01:41 -0800888 int tag;
889 int offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700890
891 height = root->height;
892 if (index > radix_tree_maxindex(height))
893 goto out;
894
Nick Piggin612d6c12006-06-23 02:03:22 -0700895 slot = root->rnode;
896 if (height == 0 && root->rnode) {
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800897 slot = radix_tree_direct_to_ptr(slot);
Nick Piggin612d6c12006-06-23 02:03:22 -0700898 root_tag_clear_all(root);
899 root->rnode = NULL;
900 goto out;
901 }
902
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903 shift = (height - 1) * RADIX_TREE_MAP_SHIFT;
904 pathp->node = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700905
Nick Piggin612d6c12006-06-23 02:03:22 -0700906 do {
Christoph Lameter201b6262005-09-06 15:16:46 -0700907 if (slot == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908 goto out;
909
Linus Torvalds1da177e2005-04-16 15:20:36 -0700910 pathp++;
Nick Piggind5274262006-01-08 01:01:41 -0800911 offset = (index >> shift) & RADIX_TREE_MAP_MASK;
912 pathp->offset = offset;
913 pathp->node = slot;
914 slot = slot->slots[offset];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700915 shift -= RADIX_TREE_MAP_SHIFT;
Nick Piggin612d6c12006-06-23 02:03:22 -0700916 height--;
917 } while (height > 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700918
Nick Piggin612d6c12006-06-23 02:03:22 -0700919 if (slot == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700920 goto out;
921
Linus Torvalds1da177e2005-04-16 15:20:36 -0700922 /*
923 * Clear all tags associated with the just-deleted item
924 */
Jonathan Corbetdaff89f2006-03-25 03:08:05 -0800925 for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++) {
Nick Piggin612d6c12006-06-23 02:03:22 -0700926 if (tag_get(pathp->node, tag, pathp->offset))
927 radix_tree_tag_clear(root, index, tag);
Nick Piggind5274262006-01-08 01:01:41 -0800928 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800930 to_free = NULL;
Christoph Lameter201b6262005-09-06 15:16:46 -0700931 /* Now free the nodes we do not need anymore */
Nick Piggin612d6c12006-06-23 02:03:22 -0700932 while (pathp->node) {
Christoph Lameter201b6262005-09-06 15:16:46 -0700933 pathp->node->slots[pathp->offset] = NULL;
Nick Piggina5f51c92006-01-08 01:01:41 -0800934 pathp->node->count--;
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800935 /*
936 * Queue the node for deferred freeing after the
937 * last reference to it disappears (set NULL, above).
938 */
939 if (to_free)
940 radix_tree_node_free(to_free);
Nick Piggina5f51c92006-01-08 01:01:41 -0800941
942 if (pathp->node->count) {
943 if (pathp->node == root->rnode)
944 radix_tree_shrink(root);
Christoph Lameter201b6262005-09-06 15:16:46 -0700945 goto out;
Nick Piggina5f51c92006-01-08 01:01:41 -0800946 }
Christoph Lameter201b6262005-09-06 15:16:46 -0700947
948 /* Node with zero slots in use so free it */
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800949 to_free = pathp->node;
Nick Piggin612d6c12006-06-23 02:03:22 -0700950 pathp--;
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800951
Linus Torvalds1da177e2005-04-16 15:20:36 -0700952 }
Nick Piggin612d6c12006-06-23 02:03:22 -0700953 root_tag_clear_all(root);
Christoph Lameter201b6262005-09-06 15:16:46 -0700954 root->height = 0;
Nick Piggin612d6c12006-06-23 02:03:22 -0700955 root->rnode = NULL;
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800956 if (to_free)
957 radix_tree_node_free(to_free);
Nick Piggin612d6c12006-06-23 02:03:22 -0700958
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959out:
Nick Piggin612d6c12006-06-23 02:03:22 -0700960 return slot;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700961}
962EXPORT_SYMBOL(radix_tree_delete);
963
964/**
965 * radix_tree_tagged - test whether any items in the tree are tagged
966 * @root: radix tree root
967 * @tag: tag to test
968 */
Jonathan Corbetdaff89f2006-03-25 03:08:05 -0800969int radix_tree_tagged(struct radix_tree_root *root, unsigned int tag)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700970{
Nick Piggin612d6c12006-06-23 02:03:22 -0700971 return root_tag_get(root, tag);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700972}
973EXPORT_SYMBOL(radix_tree_tagged);
974
975static void
Christoph Lametere18b8902006-12-06 20:33:20 -0800976radix_tree_node_ctor(void *node, struct kmem_cache *cachep, unsigned long flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700977{
978 memset(node, 0, sizeof(struct radix_tree_node));
979}
980
981static __init unsigned long __maxindex(unsigned int height)
982{
983 unsigned int tmp = height * RADIX_TREE_MAP_SHIFT;
984 unsigned long index = (~0UL >> (RADIX_TREE_INDEX_BITS - tmp - 1)) >> 1;
985
986 if (tmp >= RADIX_TREE_INDEX_BITS)
987 index = ~0UL;
988 return index;
989}
990
991static __init void radix_tree_init_maxindex(void)
992{
993 unsigned int i;
994
995 for (i = 0; i < ARRAY_SIZE(height_to_maxindex); i++)
996 height_to_maxindex[i] = __maxindex(i);
997}
998
Linus Torvalds1da177e2005-04-16 15:20:36 -0700999static int radix_tree_callback(struct notifier_block *nfb,
1000 unsigned long action,
1001 void *hcpu)
1002{
1003 int cpu = (long)hcpu;
1004 struct radix_tree_preload *rtp;
1005
1006 /* Free per-cpu pool of perloaded nodes */
1007 if (action == CPU_DEAD) {
1008 rtp = &per_cpu(radix_tree_preloads, cpu);
1009 while (rtp->nr) {
1010 kmem_cache_free(radix_tree_node_cachep,
1011 rtp->nodes[rtp->nr-1]);
1012 rtp->nodes[rtp->nr-1] = NULL;
1013 rtp->nr--;
1014 }
1015 }
1016 return NOTIFY_OK;
1017}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001018
1019void __init radix_tree_init(void)
1020{
1021 radix_tree_node_cachep = kmem_cache_create("radix_tree_node",
1022 sizeof(struct radix_tree_node), 0,
1023 SLAB_PANIC, radix_tree_node_ctor, NULL);
1024 radix_tree_init_maxindex();
1025 hotcpu_notifier(radix_tree_callback, 0);
1026}