blob: 99ab428bcfa089822e74b433aee7b1bf4076e34d [file] [log] [blame]
Alex Tomasc9de5602008-01-29 00:19:52 -05001/*
2 * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com
3 * Written by Alex Tomas <alex@clusterfs.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public Licens
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-
17 */
18
19
20/*
21 * mballoc.c contains the multiblocks allocation routines
22 */
23
Bobi Jam18aadd42012-02-20 17:53:02 -050024#include "ext4_jbd2.h"
Mingming Cao8f6e39a2008-04-29 22:01:31 -040025#include "mballoc.h"
Theodore Ts'o6ba495e2009-09-18 13:38:55 -040026#include <linux/debugfs.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090027#include <linux/slab.h>
Theodore Ts'o9bffad12009-06-17 11:48:11 -040028#include <trace/events/ext4.h>
29
Alex Tomasc9de5602008-01-29 00:19:52 -050030/*
31 * MUSTDO:
32 * - test ext4_ext_search_left() and ext4_ext_search_right()
33 * - search for metadata in few groups
34 *
35 * TODO v4:
36 * - normalization should take into account whether file is still open
37 * - discard preallocations if no free space left (policy?)
38 * - don't normalize tails
39 * - quota
40 * - reservation for superuser
41 *
42 * TODO v3:
43 * - bitmap read-ahead (proposed by Oleg Drokin aka green)
44 * - track min/max extents in each group for better group selection
45 * - mb_mark_used() may allocate chunk right after splitting buddy
46 * - tree of groups sorted by number of free blocks
47 * - error handling
48 */
49
50/*
51 * The allocation request involve request for multiple number of blocks
52 * near to the goal(block) value specified.
53 *
Theodore Ts'ob713a5e2009-03-31 09:11:14 -040054 * During initialization phase of the allocator we decide to use the
55 * group preallocation or inode preallocation depending on the size of
56 * the file. The size of the file could be the resulting file size we
57 * would have after allocation, or the current file size, which ever
58 * is larger. If the size is less than sbi->s_mb_stream_request we
59 * select to use the group preallocation. The default value of
60 * s_mb_stream_request is 16 blocks. This can also be tuned via
61 * /sys/fs/ext4/<partition>/mb_stream_req. The value is represented in
62 * terms of number of blocks.
Alex Tomasc9de5602008-01-29 00:19:52 -050063 *
64 * The main motivation for having small file use group preallocation is to
Theodore Ts'ob713a5e2009-03-31 09:11:14 -040065 * ensure that we have small files closer together on the disk.
Alex Tomasc9de5602008-01-29 00:19:52 -050066 *
Theodore Ts'ob713a5e2009-03-31 09:11:14 -040067 * First stage the allocator looks at the inode prealloc list,
68 * ext4_inode_info->i_prealloc_list, which contains list of prealloc
69 * spaces for this particular inode. The inode prealloc space is
70 * represented as:
Alex Tomasc9de5602008-01-29 00:19:52 -050071 *
72 * pa_lstart -> the logical start block for this prealloc space
73 * pa_pstart -> the physical start block for this prealloc space
Theodore Ts'o53accfa2011-09-09 18:48:51 -040074 * pa_len -> length for this prealloc space (in clusters)
75 * pa_free -> free space available in this prealloc space (in clusters)
Alex Tomasc9de5602008-01-29 00:19:52 -050076 *
77 * The inode preallocation space is used looking at the _logical_ start
78 * block. If only the logical file block falls within the range of prealloc
Tao Macaaf7a22011-07-11 18:42:42 -040079 * space we will consume the particular prealloc space. This makes sure that
80 * we have contiguous physical blocks representing the file blocks
Alex Tomasc9de5602008-01-29 00:19:52 -050081 *
82 * The important thing to be noted in case of inode prealloc space is that
83 * we don't modify the values associated to inode prealloc space except
84 * pa_free.
85 *
86 * If we are not able to find blocks in the inode prealloc space and if we
87 * have the group allocation flag set then we look at the locality group
Tao Macaaf7a22011-07-11 18:42:42 -040088 * prealloc space. These are per CPU prealloc list represented as
Alex Tomasc9de5602008-01-29 00:19:52 -050089 *
90 * ext4_sb_info.s_locality_groups[smp_processor_id()]
91 *
92 * The reason for having a per cpu locality group is to reduce the contention
93 * between CPUs. It is possible to get scheduled at this point.
94 *
95 * The locality group prealloc space is used looking at whether we have
Lucas De Marchi25985ed2011-03-30 22:57:33 -030096 * enough free space (pa_free) within the prealloc space.
Alex Tomasc9de5602008-01-29 00:19:52 -050097 *
98 * If we can't allocate blocks via inode prealloc or/and locality group
99 * prealloc then we look at the buddy cache. The buddy cache is represented
100 * by ext4_sb_info.s_buddy_cache (struct inode) whose file offset gets
101 * mapped to the buddy and bitmap information regarding different
102 * groups. The buddy information is attached to buddy cache inode so that
103 * we can access them through the page cache. The information regarding
104 * each group is loaded via ext4_mb_load_buddy. The information involve
105 * block bitmap and buddy information. The information are stored in the
106 * inode as:
107 *
108 * { page }
Aneesh Kumar K.Vc3a326a2008-11-25 15:11:52 -0500109 * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]...
Alex Tomasc9de5602008-01-29 00:19:52 -0500110 *
111 *
112 * one block each for bitmap and buddy information. So for each group we
113 * take up 2 blocks. A page can contain blocks_per_page (PAGE_CACHE_SIZE /
114 * blocksize) blocks. So it can have information regarding groups_per_page
115 * which is blocks_per_page/2
116 *
117 * The buddy cache inode is not stored on disk. The inode is thrown
118 * away when the filesystem is unmounted.
119 *
120 * We look for count number of blocks in the buddy cache. If we were able
121 * to locate that many free blocks we return with additional information
122 * regarding rest of the contiguous physical block available
123 *
124 * Before allocating blocks via buddy cache we normalize the request
125 * blocks. This ensure we ask for more blocks that we needed. The extra
126 * blocks that we get after allocation is added to the respective prealloc
127 * list. In case of inode preallocation we follow a list of heuristics
128 * based on file size. This can be found in ext4_mb_normalize_request. If
129 * we are doing a group prealloc we try to normalize the request to
Theodore Ts'o27baebb2011-09-09 19:02:51 -0400130 * sbi->s_mb_group_prealloc. The default value of s_mb_group_prealloc is
131 * dependent on the cluster size; for non-bigalloc file systems, it is
Alex Tomasc9de5602008-01-29 00:19:52 -0500132 * 512 blocks. This can be tuned via
Dan Ehrenbergd7a1fee2011-07-17 21:11:30 -0400133 * /sys/fs/ext4/<partition>/mb_group_prealloc. The value is represented in
Alex Tomasc9de5602008-01-29 00:19:52 -0500134 * terms of number of blocks. If we have mounted the file system with -O
135 * stripe=<value> option the group prealloc request is normalized to the
Dan Ehrenbergd7a1fee2011-07-17 21:11:30 -0400136 * the smallest multiple of the stripe value (sbi->s_stripe) which is
137 * greater than the default mb_group_prealloc.
Alex Tomasc9de5602008-01-29 00:19:52 -0500138 *
Dan Ehrenbergd7a1fee2011-07-17 21:11:30 -0400139 * The regular allocator (using the buddy cache) supports a few tunables.
Alex Tomasc9de5602008-01-29 00:19:52 -0500140 *
Theodore Ts'ob713a5e2009-03-31 09:11:14 -0400141 * /sys/fs/ext4/<partition>/mb_min_to_scan
142 * /sys/fs/ext4/<partition>/mb_max_to_scan
143 * /sys/fs/ext4/<partition>/mb_order2_req
Alex Tomasc9de5602008-01-29 00:19:52 -0500144 *
Theodore Ts'ob713a5e2009-03-31 09:11:14 -0400145 * The regular allocator uses buddy scan only if the request len is power of
Alex Tomasc9de5602008-01-29 00:19:52 -0500146 * 2 blocks and the order of allocation is >= sbi->s_mb_order2_reqs. The
147 * value of s_mb_order2_reqs can be tuned via
Theodore Ts'ob713a5e2009-03-31 09:11:14 -0400148 * /sys/fs/ext4/<partition>/mb_order2_req. If the request len is equal to
André Goddard Rosaaf901ca2009-11-14 13:09:05 -0200149 * stripe size (sbi->s_stripe), we try to search for contiguous block in
Theodore Ts'ob713a5e2009-03-31 09:11:14 -0400150 * stripe size. This should result in better allocation on RAID setups. If
151 * not, we search in the specific group using bitmap for best extents. The
152 * tunable min_to_scan and max_to_scan control the behaviour here.
Alex Tomasc9de5602008-01-29 00:19:52 -0500153 * min_to_scan indicate how long the mballoc __must__ look for a best
Theodore Ts'ob713a5e2009-03-31 09:11:14 -0400154 * extent and max_to_scan indicates how long the mballoc __can__ look for a
Alex Tomasc9de5602008-01-29 00:19:52 -0500155 * best extent in the found extents. Searching for the blocks starts with
156 * the group specified as the goal value in allocation context via
157 * ac_g_ex. Each group is first checked based on the criteria whether it
Tao Macaaf7a22011-07-11 18:42:42 -0400158 * can be used for allocation. ext4_mb_good_group explains how the groups are
Alex Tomasc9de5602008-01-29 00:19:52 -0500159 * checked.
160 *
161 * Both the prealloc space are getting populated as above. So for the first
162 * request we will hit the buddy cache which will result in this prealloc
163 * space getting filled. The prealloc space is then later used for the
164 * subsequent request.
165 */
166
167/*
168 * mballoc operates on the following data:
169 * - on-disk bitmap
170 * - in-core buddy (actually includes buddy and bitmap)
171 * - preallocation descriptors (PAs)
172 *
173 * there are two types of preallocations:
174 * - inode
175 * assiged to specific inode and can be used for this inode only.
176 * it describes part of inode's space preallocated to specific
177 * physical blocks. any block from that preallocated can be used
178 * independent. the descriptor just tracks number of blocks left
179 * unused. so, before taking some block from descriptor, one must
180 * make sure corresponded logical block isn't allocated yet. this
181 * also means that freeing any block within descriptor's range
182 * must discard all preallocated blocks.
183 * - locality group
184 * assigned to specific locality group which does not translate to
185 * permanent set of inodes: inode can join and leave group. space
186 * from this type of preallocation can be used for any inode. thus
187 * it's consumed from the beginning to the end.
188 *
189 * relation between them can be expressed as:
190 * in-core buddy = on-disk bitmap + preallocation descriptors
191 *
192 * this mean blocks mballoc considers used are:
193 * - allocated blocks (persistent)
194 * - preallocated blocks (non-persistent)
195 *
196 * consistency in mballoc world means that at any time a block is either
197 * free or used in ALL structures. notice: "any time" should not be read
198 * literally -- time is discrete and delimited by locks.
199 *
200 * to keep it simple, we don't use block numbers, instead we count number of
201 * blocks: how many blocks marked used/free in on-disk bitmap, buddy and PA.
202 *
203 * all operations can be expressed as:
204 * - init buddy: buddy = on-disk + PAs
205 * - new PA: buddy += N; PA = N
206 * - use inode PA: on-disk += N; PA -= N
207 * - discard inode PA buddy -= on-disk - PA; PA = 0
208 * - use locality group PA on-disk += N; PA -= N
209 * - discard locality group PA buddy -= PA; PA = 0
210 * note: 'buddy -= on-disk - PA' is used to show that on-disk bitmap
211 * is used in real operation because we can't know actual used
212 * bits from PA, only from on-disk bitmap
213 *
214 * if we follow this strict logic, then all operations above should be atomic.
215 * given some of them can block, we'd have to use something like semaphores
216 * killing performance on high-end SMP hardware. let's try to relax it using
217 * the following knowledge:
218 * 1) if buddy is referenced, it's already initialized
219 * 2) while block is used in buddy and the buddy is referenced,
220 * nobody can re-allocate that block
221 * 3) we work on bitmaps and '+' actually means 'set bits'. if on-disk has
222 * bit set and PA claims same block, it's OK. IOW, one can set bit in
223 * on-disk bitmap if buddy has same bit set or/and PA covers corresponded
224 * block
225 *
226 * so, now we're building a concurrency table:
227 * - init buddy vs.
228 * - new PA
229 * blocks for PA are allocated in the buddy, buddy must be referenced
230 * until PA is linked to allocation group to avoid concurrent buddy init
231 * - use inode PA
232 * we need to make sure that either on-disk bitmap or PA has uptodate data
233 * given (3) we care that PA-=N operation doesn't interfere with init
234 * - discard inode PA
235 * the simplest way would be to have buddy initialized by the discard
236 * - use locality group PA
237 * again PA-=N must be serialized with init
238 * - discard locality group PA
239 * the simplest way would be to have buddy initialized by the discard
240 * - new PA vs.
241 * - use inode PA
242 * i_data_sem serializes them
243 * - discard inode PA
244 * discard process must wait until PA isn't used by another process
245 * - use locality group PA
246 * some mutex should serialize them
247 * - discard locality group PA
248 * discard process must wait until PA isn't used by another process
249 * - use inode PA
250 * - use inode PA
251 * i_data_sem or another mutex should serializes them
252 * - discard inode PA
253 * discard process must wait until PA isn't used by another process
254 * - use locality group PA
255 * nothing wrong here -- they're different PAs covering different blocks
256 * - discard locality group PA
257 * discard process must wait until PA isn't used by another process
258 *
259 * now we're ready to make few consequences:
260 * - PA is referenced and while it is no discard is possible
261 * - PA is referenced until block isn't marked in on-disk bitmap
262 * - PA changes only after on-disk bitmap
263 * - discard must not compete with init. either init is done before
264 * any discard or they're serialized somehow
265 * - buddy init as sum of on-disk bitmap and PAs is done atomically
266 *
267 * a special case when we've used PA to emptiness. no need to modify buddy
268 * in this case, but we should care about concurrent init
269 *
270 */
271
272 /*
273 * Logic in few words:
274 *
275 * - allocation:
276 * load group
277 * find blocks
278 * mark bits in on-disk bitmap
279 * release group
280 *
281 * - use preallocation:
282 * find proper PA (per-inode or group)
283 * load group
284 * mark bits in on-disk bitmap
285 * release group
286 * release PA
287 *
288 * - free:
289 * load group
290 * mark bits in on-disk bitmap
291 * release group
292 *
293 * - discard preallocations in group:
294 * mark PAs deleted
295 * move them onto local list
296 * load on-disk bitmap
297 * load group
298 * remove PA from object (inode or locality group)
299 * mark free blocks in-core
300 *
301 * - discard inode's preallocations:
302 */
303
304/*
305 * Locking rules
306 *
307 * Locks:
308 * - bitlock on a group (group)
309 * - object (inode/locality) (object)
310 * - per-pa lock (pa)
311 *
312 * Paths:
313 * - new pa
314 * object
315 * group
316 *
317 * - find and use pa:
318 * pa
319 *
320 * - release consumed pa:
321 * pa
322 * group
323 * object
324 *
325 * - generate in-core bitmap:
326 * group
327 * pa
328 *
329 * - discard all for given object (inode, locality group):
330 * object
331 * pa
332 * group
333 *
334 * - discard all for given group:
335 * group
336 * pa
337 * group
338 * object
339 *
340 */
Aneesh Kumar K.Vc3a326a2008-11-25 15:11:52 -0500341static struct kmem_cache *ext4_pspace_cachep;
342static struct kmem_cache *ext4_ac_cachep;
Bobi Jam18aadd42012-02-20 17:53:02 -0500343static struct kmem_cache *ext4_free_data_cachep;
Curt Wohlgemuthfb1813f2010-10-27 21:29:12 -0400344
345/* We create slab caches for groupinfo data structures based on the
346 * superblock block size. There will be one per mounted filesystem for
347 * each unique s_blocksize_bits */
Eric Sandeen2892c152011-02-12 08:12:18 -0500348#define NR_GRPINFO_CACHES 8
Curt Wohlgemuthfb1813f2010-10-27 21:29:12 -0400349static struct kmem_cache *ext4_groupinfo_caches[NR_GRPINFO_CACHES];
350
Eric Sandeen2892c152011-02-12 08:12:18 -0500351static const char *ext4_groupinfo_slab_names[NR_GRPINFO_CACHES] = {
352 "ext4_groupinfo_1k", "ext4_groupinfo_2k", "ext4_groupinfo_4k",
353 "ext4_groupinfo_8k", "ext4_groupinfo_16k", "ext4_groupinfo_32k",
354 "ext4_groupinfo_64k", "ext4_groupinfo_128k"
355};
356
Aneesh Kumar K.Vc3a326a2008-11-25 15:11:52 -0500357static void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap,
358 ext4_group_t group);
Aneesh Kumar K.V7a2fcbf2009-01-05 21:36:55 -0500359static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap,
360 ext4_group_t group);
Bobi Jam18aadd42012-02-20 17:53:02 -0500361static void ext4_free_data_callback(struct super_block *sb,
362 struct ext4_journal_cb_entry *jce, int rc);
Aneesh Kumar K.Vc3a326a2008-11-25 15:11:52 -0500363
Aneesh Kumar K.Vffad0a42008-02-23 01:38:34 -0500364static inline void *mb_correct_addr_and_bit(int *bit, void *addr)
365{
Alex Tomasc9de5602008-01-29 00:19:52 -0500366#if BITS_PER_LONG == 64
Aneesh Kumar K.Vffad0a42008-02-23 01:38:34 -0500367 *bit += ((unsigned long) addr & 7UL) << 3;
368 addr = (void *) ((unsigned long) addr & ~7UL);
Alex Tomasc9de5602008-01-29 00:19:52 -0500369#elif BITS_PER_LONG == 32
Aneesh Kumar K.Vffad0a42008-02-23 01:38:34 -0500370 *bit += ((unsigned long) addr & 3UL) << 3;
371 addr = (void *) ((unsigned long) addr & ~3UL);
Alex Tomasc9de5602008-01-29 00:19:52 -0500372#else
373#error "how many bits you are?!"
374#endif
Aneesh Kumar K.Vffad0a42008-02-23 01:38:34 -0500375 return addr;
376}
Alex Tomasc9de5602008-01-29 00:19:52 -0500377
378static inline int mb_test_bit(int bit, void *addr)
379{
380 /*
381 * ext4_test_bit on architecture like powerpc
382 * needs unsigned long aligned address
383 */
Aneesh Kumar K.Vffad0a42008-02-23 01:38:34 -0500384 addr = mb_correct_addr_and_bit(&bit, addr);
Alex Tomasc9de5602008-01-29 00:19:52 -0500385 return ext4_test_bit(bit, addr);
386}
387
388static inline void mb_set_bit(int bit, void *addr)
389{
Aneesh Kumar K.Vffad0a42008-02-23 01:38:34 -0500390 addr = mb_correct_addr_and_bit(&bit, addr);
Alex Tomasc9de5602008-01-29 00:19:52 -0500391 ext4_set_bit(bit, addr);
392}
393
Alex Tomasc9de5602008-01-29 00:19:52 -0500394static inline void mb_clear_bit(int bit, void *addr)
395{
Aneesh Kumar K.Vffad0a42008-02-23 01:38:34 -0500396 addr = mb_correct_addr_and_bit(&bit, addr);
Alex Tomasc9de5602008-01-29 00:19:52 -0500397 ext4_clear_bit(bit, addr);
398}
399
Aneesh Kumar K.Vffad0a42008-02-23 01:38:34 -0500400static inline int mb_find_next_zero_bit(void *addr, int max, int start)
401{
Aneesh Kumar K.Ve7dfb242008-07-11 19:27:31 -0400402 int fix = 0, ret, tmpmax;
Aneesh Kumar K.Vffad0a42008-02-23 01:38:34 -0500403 addr = mb_correct_addr_and_bit(&fix, addr);
Aneesh Kumar K.Ve7dfb242008-07-11 19:27:31 -0400404 tmpmax = max + fix;
Aneesh Kumar K.Vffad0a42008-02-23 01:38:34 -0500405 start += fix;
406
Aneesh Kumar K.Ve7dfb242008-07-11 19:27:31 -0400407 ret = ext4_find_next_zero_bit(addr, tmpmax, start) - fix;
408 if (ret > max)
409 return max;
410 return ret;
Aneesh Kumar K.Vffad0a42008-02-23 01:38:34 -0500411}
412
413static inline int mb_find_next_bit(void *addr, int max, int start)
414{
Aneesh Kumar K.Ve7dfb242008-07-11 19:27:31 -0400415 int fix = 0, ret, tmpmax;
Aneesh Kumar K.Vffad0a42008-02-23 01:38:34 -0500416 addr = mb_correct_addr_and_bit(&fix, addr);
Aneesh Kumar K.Ve7dfb242008-07-11 19:27:31 -0400417 tmpmax = max + fix;
Aneesh Kumar K.Vffad0a42008-02-23 01:38:34 -0500418 start += fix;
419
Aneesh Kumar K.Ve7dfb242008-07-11 19:27:31 -0400420 ret = ext4_find_next_bit(addr, tmpmax, start) - fix;
421 if (ret > max)
422 return max;
423 return ret;
Aneesh Kumar K.Vffad0a42008-02-23 01:38:34 -0500424}
425
Alex Tomasc9de5602008-01-29 00:19:52 -0500426static void *mb_find_buddy(struct ext4_buddy *e4b, int order, int *max)
427{
428 char *bb;
429
Theodore Ts'oc5e8f3f2012-02-20 17:54:06 -0500430 BUG_ON(e4b->bd_bitmap == e4b->bd_buddy);
Alex Tomasc9de5602008-01-29 00:19:52 -0500431 BUG_ON(max == NULL);
432
433 if (order > e4b->bd_blkbits + 1) {
434 *max = 0;
435 return NULL;
436 }
437
438 /* at order 0 we see each particular block */
Coly Li84b775a2011-02-24 12:51:59 -0500439 if (order == 0) {
440 *max = 1 << (e4b->bd_blkbits + 3);
Theodore Ts'oc5e8f3f2012-02-20 17:54:06 -0500441 return e4b->bd_bitmap;
Coly Li84b775a2011-02-24 12:51:59 -0500442 }
Alex Tomasc9de5602008-01-29 00:19:52 -0500443
Theodore Ts'oc5e8f3f2012-02-20 17:54:06 -0500444 bb = e4b->bd_buddy + EXT4_SB(e4b->bd_sb)->s_mb_offsets[order];
Alex Tomasc9de5602008-01-29 00:19:52 -0500445 *max = EXT4_SB(e4b->bd_sb)->s_mb_maxs[order];
446
447 return bb;
448}
449
450#ifdef DOUBLE_CHECK
451static void mb_free_blocks_double(struct inode *inode, struct ext4_buddy *e4b,
452 int first, int count)
453{
454 int i;
455 struct super_block *sb = e4b->bd_sb;
456
457 if (unlikely(e4b->bd_info->bb_bitmap == NULL))
458 return;
Vincent Minetbc8e6742009-05-15 08:33:18 -0400459 assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group));
Alex Tomasc9de5602008-01-29 00:19:52 -0500460 for (i = 0; i < count; i++) {
461 if (!mb_test_bit(first + i, e4b->bd_info->bb_bitmap)) {
462 ext4_fsblk_t blocknr;
Akinobu Mita5661bd62010-03-03 23:53:39 -0500463
464 blocknr = ext4_group_first_block_no(sb, e4b->bd_group);
Theodore Ts'o53accfa2011-09-09 18:48:51 -0400465 blocknr += EXT4_C2B(EXT4_SB(sb), first + i);
Aneesh Kumar K.V5d1b1b32009-01-05 22:19:52 -0500466 ext4_grp_locked_error(sb, e4b->bd_group,
Theodore Ts'oe29136f2010-06-29 12:54:28 -0400467 inode ? inode->i_ino : 0,
468 blocknr,
469 "freeing block already freed "
470 "(bit %u)",
471 first + i);
Alex Tomasc9de5602008-01-29 00:19:52 -0500472 }
473 mb_clear_bit(first + i, e4b->bd_info->bb_bitmap);
474 }
475}
476
477static void mb_mark_used_double(struct ext4_buddy *e4b, int first, int count)
478{
479 int i;
480
481 if (unlikely(e4b->bd_info->bb_bitmap == NULL))
482 return;
Vincent Minetbc8e6742009-05-15 08:33:18 -0400483 assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));
Alex Tomasc9de5602008-01-29 00:19:52 -0500484 for (i = 0; i < count; i++) {
485 BUG_ON(mb_test_bit(first + i, e4b->bd_info->bb_bitmap));
486 mb_set_bit(first + i, e4b->bd_info->bb_bitmap);
487 }
488}
489
490static void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap)
491{
492 if (memcmp(e4b->bd_info->bb_bitmap, bitmap, e4b->bd_sb->s_blocksize)) {
493 unsigned char *b1, *b2;
494 int i;
495 b1 = (unsigned char *) e4b->bd_info->bb_bitmap;
496 b2 = (unsigned char *) bitmap;
497 for (i = 0; i < e4b->bd_sb->s_blocksize; i++) {
498 if (b1[i] != b2[i]) {
Theodore Ts'o9d8b9ec2011-08-01 17:41:35 -0400499 ext4_msg(e4b->bd_sb, KERN_ERR,
500 "corruption in group %u "
501 "at byte %u(%u): %x in copy != %x "
502 "on disk/prealloc",
503 e4b->bd_group, i, i * 8, b1[i], b2[i]);
Alex Tomasc9de5602008-01-29 00:19:52 -0500504 BUG();
505 }
506 }
507 }
508}
509
510#else
511static inline void mb_free_blocks_double(struct inode *inode,
512 struct ext4_buddy *e4b, int first, int count)
513{
514 return;
515}
516static inline void mb_mark_used_double(struct ext4_buddy *e4b,
517 int first, int count)
518{
519 return;
520}
521static inline void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap)
522{
523 return;
524}
525#endif
526
527#ifdef AGGRESSIVE_CHECK
528
529#define MB_CHECK_ASSERT(assert) \
530do { \
531 if (!(assert)) { \
532 printk(KERN_EMERG \
533 "Assertion failure in %s() at %s:%d: \"%s\"\n", \
534 function, file, line, # assert); \
535 BUG(); \
536 } \
537} while (0)
538
539static int __mb_check_buddy(struct ext4_buddy *e4b, char *file,
540 const char *function, int line)
541{
542 struct super_block *sb = e4b->bd_sb;
543 int order = e4b->bd_blkbits + 1;
544 int max;
545 int max2;
546 int i;
547 int j;
548 int k;
549 int count;
550 struct ext4_group_info *grp;
551 int fragments = 0;
552 int fstart;
553 struct list_head *cur;
554 void *buddy;
555 void *buddy2;
556
Alex Tomasc9de5602008-01-29 00:19:52 -0500557 {
558 static int mb_check_counter;
559 if (mb_check_counter++ % 100 != 0)
560 return 0;
561 }
562
563 while (order > 1) {
564 buddy = mb_find_buddy(e4b, order, &max);
565 MB_CHECK_ASSERT(buddy);
566 buddy2 = mb_find_buddy(e4b, order - 1, &max2);
567 MB_CHECK_ASSERT(buddy2);
568 MB_CHECK_ASSERT(buddy != buddy2);
569 MB_CHECK_ASSERT(max * 2 == max2);
570
571 count = 0;
572 for (i = 0; i < max; i++) {
573
574 if (mb_test_bit(i, buddy)) {
575 /* only single bit in buddy2 may be 1 */
576 if (!mb_test_bit(i << 1, buddy2)) {
577 MB_CHECK_ASSERT(
578 mb_test_bit((i<<1)+1, buddy2));
579 } else if (!mb_test_bit((i << 1) + 1, buddy2)) {
580 MB_CHECK_ASSERT(
581 mb_test_bit(i << 1, buddy2));
582 }
583 continue;
584 }
585
Robin Dong0a10da72011-10-26 08:48:54 -0400586 /* both bits in buddy2 must be 1 */
Alex Tomasc9de5602008-01-29 00:19:52 -0500587 MB_CHECK_ASSERT(mb_test_bit(i << 1, buddy2));
588 MB_CHECK_ASSERT(mb_test_bit((i << 1) + 1, buddy2));
589
590 for (j = 0; j < (1 << order); j++) {
591 k = (i * (1 << order)) + j;
592 MB_CHECK_ASSERT(
Theodore Ts'oc5e8f3f2012-02-20 17:54:06 -0500593 !mb_test_bit(k, e4b->bd_bitmap));
Alex Tomasc9de5602008-01-29 00:19:52 -0500594 }
595 count++;
596 }
597 MB_CHECK_ASSERT(e4b->bd_info->bb_counters[order] == count);
598 order--;
599 }
600
601 fstart = -1;
602 buddy = mb_find_buddy(e4b, 0, &max);
603 for (i = 0; i < max; i++) {
604 if (!mb_test_bit(i, buddy)) {
605 MB_CHECK_ASSERT(i >= e4b->bd_info->bb_first_free);
606 if (fstart == -1) {
607 fragments++;
608 fstart = i;
609 }
610 continue;
611 }
612 fstart = -1;
613 /* check used bits only */
614 for (j = 0; j < e4b->bd_blkbits + 1; j++) {
615 buddy2 = mb_find_buddy(e4b, j, &max2);
616 k = i >> j;
617 MB_CHECK_ASSERT(k < max2);
618 MB_CHECK_ASSERT(mb_test_bit(k, buddy2));
619 }
620 }
621 MB_CHECK_ASSERT(!EXT4_MB_GRP_NEED_INIT(e4b->bd_info));
622 MB_CHECK_ASSERT(e4b->bd_info->bb_fragments == fragments);
623
624 grp = ext4_get_group_info(sb, e4b->bd_group);
Alex Tomasc9de5602008-01-29 00:19:52 -0500625 list_for_each(cur, &grp->bb_prealloc_list) {
626 ext4_group_t groupnr;
627 struct ext4_prealloc_space *pa;
Solofo Ramangalahy60bd63d2008-04-29 21:59:59 -0400628 pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list);
629 ext4_get_group_no_and_offset(sb, pa->pa_pstart, &groupnr, &k);
Alex Tomasc9de5602008-01-29 00:19:52 -0500630 MB_CHECK_ASSERT(groupnr == e4b->bd_group);
Solofo Ramangalahy60bd63d2008-04-29 21:59:59 -0400631 for (i = 0; i < pa->pa_len; i++)
Alex Tomasc9de5602008-01-29 00:19:52 -0500632 MB_CHECK_ASSERT(mb_test_bit(k + i, buddy));
633 }
634 return 0;
635}
636#undef MB_CHECK_ASSERT
637#define mb_check_buddy(e4b) __mb_check_buddy(e4b, \
Harvey Harrison46e665e2008-04-17 10:38:59 -0400638 __FILE__, __func__, __LINE__)
Alex Tomasc9de5602008-01-29 00:19:52 -0500639#else
640#define mb_check_buddy(e4b)
641#endif
642
Coly Li7c786052011-02-24 13:24:25 -0500643/*
644 * Divide blocks started from @first with length @len into
645 * smaller chunks with power of 2 blocks.
646 * Clear the bits in bitmap which the blocks of the chunk(s) covered,
647 * then increase bb_counters[] for corresponded chunk size.
648 */
Alex Tomasc9de5602008-01-29 00:19:52 -0500649static void ext4_mb_mark_free_simple(struct super_block *sb,
Eric Sandeena36b4492009-08-25 22:36:45 -0400650 void *buddy, ext4_grpblk_t first, ext4_grpblk_t len,
Alex Tomasc9de5602008-01-29 00:19:52 -0500651 struct ext4_group_info *grp)
652{
653 struct ext4_sb_info *sbi = EXT4_SB(sb);
Eric Sandeena36b4492009-08-25 22:36:45 -0400654 ext4_grpblk_t min;
655 ext4_grpblk_t max;
656 ext4_grpblk_t chunk;
Alex Tomasc9de5602008-01-29 00:19:52 -0500657 unsigned short border;
658
Theodore Ts'o7137d7a2011-09-09 18:38:51 -0400659 BUG_ON(len > EXT4_CLUSTERS_PER_GROUP(sb));
Alex Tomasc9de5602008-01-29 00:19:52 -0500660
661 border = 2 << sb->s_blocksize_bits;
662
663 while (len > 0) {
664 /* find how many blocks can be covered since this position */
665 max = ffs(first | border) - 1;
666
667 /* find how many blocks of power 2 we need to mark */
668 min = fls(len) - 1;
669
670 if (max < min)
671 min = max;
672 chunk = 1 << min;
673
674 /* mark multiblock chunks only */
675 grp->bb_counters[min]++;
676 if (min > 0)
677 mb_clear_bit(first >> min,
678 buddy + sbi->s_mb_offsets[min]);
679
680 len -= chunk;
681 first += chunk;
682 }
683}
684
Curt Wohlgemuth8a57d9d2010-05-16 15:00:00 -0400685/*
686 * Cache the order of the largest free extent we have available in this block
687 * group.
688 */
689static void
690mb_set_largest_free_order(struct super_block *sb, struct ext4_group_info *grp)
691{
692 int i;
693 int bits;
694
695 grp->bb_largest_free_order = -1; /* uninit */
696
697 bits = sb->s_blocksize_bits + 1;
698 for (i = bits; i >= 0; i--) {
699 if (grp->bb_counters[i] > 0) {
700 grp->bb_largest_free_order = i;
701 break;
702 }
703 }
704}
705
Eric Sandeen089ceec2009-07-05 22:17:31 -0400706static noinline_for_stack
707void ext4_mb_generate_buddy(struct super_block *sb,
Alex Tomasc9de5602008-01-29 00:19:52 -0500708 void *buddy, void *bitmap, ext4_group_t group)
709{
710 struct ext4_group_info *grp = ext4_get_group_info(sb, group);
Theodore Ts'o7137d7a2011-09-09 18:38:51 -0400711 ext4_grpblk_t max = EXT4_CLUSTERS_PER_GROUP(sb);
Eric Sandeena36b4492009-08-25 22:36:45 -0400712 ext4_grpblk_t i = 0;
713 ext4_grpblk_t first;
714 ext4_grpblk_t len;
Alex Tomasc9de5602008-01-29 00:19:52 -0500715 unsigned free = 0;
716 unsigned fragments = 0;
717 unsigned long long period = get_cycles();
718
719 /* initialize buddy from bitmap which is aggregation
720 * of on-disk bitmap and preallocations */
Aneesh Kumar K.Vffad0a42008-02-23 01:38:34 -0500721 i = mb_find_next_zero_bit(bitmap, max, 0);
Alex Tomasc9de5602008-01-29 00:19:52 -0500722 grp->bb_first_free = i;
723 while (i < max) {
724 fragments++;
725 first = i;
Aneesh Kumar K.Vffad0a42008-02-23 01:38:34 -0500726 i = mb_find_next_bit(bitmap, max, i);
Alex Tomasc9de5602008-01-29 00:19:52 -0500727 len = i - first;
728 free += len;
729 if (len > 1)
730 ext4_mb_mark_free_simple(sb, buddy, first, len, grp);
731 else
732 grp->bb_counters[0]++;
733 if (i < max)
Aneesh Kumar K.Vffad0a42008-02-23 01:38:34 -0500734 i = mb_find_next_zero_bit(bitmap, max, i);
Alex Tomasc9de5602008-01-29 00:19:52 -0500735 }
736 grp->bb_fragments = fragments;
737
738 if (free != grp->bb_free) {
Theodore Ts'oe29136f2010-06-29 12:54:28 -0400739 ext4_grp_locked_error(sb, group, 0, 0,
Theodore Ts'o53accfa2011-09-09 18:48:51 -0400740 "%u clusters in bitmap, %u in gd",
Theodore Ts'oe29136f2010-06-29 12:54:28 -0400741 free, grp->bb_free);
Aneesh Kumar K.Ve56eb652008-02-15 13:48:21 -0500742 /*
743 * If we intent to continue, we consider group descritor
744 * corrupt and update bb_free using bitmap value
745 */
Alex Tomasc9de5602008-01-29 00:19:52 -0500746 grp->bb_free = free;
747 }
Curt Wohlgemuth8a57d9d2010-05-16 15:00:00 -0400748 mb_set_largest_free_order(sb, grp);
Alex Tomasc9de5602008-01-29 00:19:52 -0500749
750 clear_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &(grp->bb_state));
751
752 period = get_cycles() - period;
753 spin_lock(&EXT4_SB(sb)->s_bal_lock);
754 EXT4_SB(sb)->s_mb_buddies_generated++;
755 EXT4_SB(sb)->s_mb_generation_time += period;
756 spin_unlock(&EXT4_SB(sb)->s_bal_lock);
757}
758
759/* The buddy information is attached the buddy cache inode
760 * for convenience. The information regarding each group
761 * is loaded via ext4_mb_load_buddy. The information involve
762 * block bitmap and buddy information. The information are
763 * stored in the inode as
764 *
765 * { page }
Aneesh Kumar K.Vc3a326a2008-11-25 15:11:52 -0500766 * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]...
Alex Tomasc9de5602008-01-29 00:19:52 -0500767 *
768 *
769 * one block each for bitmap and buddy information.
770 * So for each group we take up 2 blocks. A page can
771 * contain blocks_per_page (PAGE_CACHE_SIZE / blocksize) blocks.
772 * So it can have information regarding groups_per_page which
773 * is blocks_per_page/2
Curt Wohlgemuth8a57d9d2010-05-16 15:00:00 -0400774 *
775 * Locking note: This routine takes the block group lock of all groups
776 * for this page; do not hold this lock when calling this routine!
Alex Tomasc9de5602008-01-29 00:19:52 -0500777 */
778
779static int ext4_mb_init_cache(struct page *page, char *incore)
780{
Theodore Ts'o8df96752009-05-01 08:50:38 -0400781 ext4_group_t ngroups;
Alex Tomasc9de5602008-01-29 00:19:52 -0500782 int blocksize;
783 int blocks_per_page;
784 int groups_per_page;
785 int err = 0;
786 int i;
Theodore Ts'o813e5722012-02-20 17:52:46 -0500787 ext4_group_t first_group, group;
Alex Tomasc9de5602008-01-29 00:19:52 -0500788 int first_block;
789 struct super_block *sb;
790 struct buffer_head *bhs;
791 struct buffer_head **bh;
792 struct inode *inode;
793 char *data;
794 char *bitmap;
Amir Goldstein9b8b7d32011-05-09 21:49:42 -0400795 struct ext4_group_info *grinfo;
Alex Tomasc9de5602008-01-29 00:19:52 -0500796
Theodore Ts'o6ba495e2009-09-18 13:38:55 -0400797 mb_debug(1, "init page %lu\n", page->index);
Alex Tomasc9de5602008-01-29 00:19:52 -0500798
799 inode = page->mapping->host;
800 sb = inode->i_sb;
Theodore Ts'o8df96752009-05-01 08:50:38 -0400801 ngroups = ext4_get_groups_count(sb);
Alex Tomasc9de5602008-01-29 00:19:52 -0500802 blocksize = 1 << inode->i_blkbits;
803 blocks_per_page = PAGE_CACHE_SIZE / blocksize;
804
805 groups_per_page = blocks_per_page >> 1;
806 if (groups_per_page == 0)
807 groups_per_page = 1;
808
809 /* allocate buffer_heads to read bitmaps */
810 if (groups_per_page > 1) {
Alex Tomasc9de5602008-01-29 00:19:52 -0500811 i = sizeof(struct buffer_head *) * groups_per_page;
812 bh = kzalloc(i, GFP_NOFS);
Theodore Ts'o813e5722012-02-20 17:52:46 -0500813 if (bh == NULL) {
814 err = -ENOMEM;
Alex Tomasc9de5602008-01-29 00:19:52 -0500815 goto out;
Theodore Ts'o813e5722012-02-20 17:52:46 -0500816 }
Alex Tomasc9de5602008-01-29 00:19:52 -0500817 } else
818 bh = &bhs;
819
820 first_group = page->index * blocks_per_page / 2;
821
822 /* read all groups the page covers into the cache */
Theodore Ts'o813e5722012-02-20 17:52:46 -0500823 for (i = 0, group = first_group; i < groups_per_page; i++, group++) {
824 if (group >= ngroups)
Alex Tomasc9de5602008-01-29 00:19:52 -0500825 break;
826
Theodore Ts'o813e5722012-02-20 17:52:46 -0500827 grinfo = ext4_get_group_info(sb, group);
Amir Goldstein9b8b7d32011-05-09 21:49:42 -0400828 /*
829 * If page is uptodate then we came here after online resize
830 * which added some new uninitialized group info structs, so
831 * we must skip all initialized uptodate buddies on the page,
832 * which may be currently in use by an allocating task.
833 */
834 if (PageUptodate(page) && !EXT4_MB_GRP_NEED_INIT(grinfo)) {
835 bh[i] = NULL;
836 continue;
837 }
Theodore Ts'o813e5722012-02-20 17:52:46 -0500838 if (!(bh[i] = ext4_read_block_bitmap_nowait(sb, group))) {
839 err = -ENOMEM;
Alex Tomasc9de5602008-01-29 00:19:52 -0500840 goto out;
Aneesh Kumar K.V2ccb5fb2009-01-05 21:49:55 -0500841 }
Theodore Ts'o813e5722012-02-20 17:52:46 -0500842 mb_debug(1, "read bitmap for group %u\n", group);
Alex Tomasc9de5602008-01-29 00:19:52 -0500843 }
844
845 /* wait for I/O completion */
Theodore Ts'o813e5722012-02-20 17:52:46 -0500846 for (i = 0, group = first_group; i < groups_per_page; i++, group++) {
847 if (bh[i] && ext4_wait_block_bitmap(sb, group, bh[i])) {
848 err = -EIO;
Alex Tomasc9de5602008-01-29 00:19:52 -0500849 goto out;
Theodore Ts'o813e5722012-02-20 17:52:46 -0500850 }
851 }
Alex Tomasc9de5602008-01-29 00:19:52 -0500852
853 first_block = page->index * blocks_per_page;
854 for (i = 0; i < blocks_per_page; i++) {
855 int group;
Alex Tomasc9de5602008-01-29 00:19:52 -0500856
857 group = (first_block + i) >> 1;
Theodore Ts'o8df96752009-05-01 08:50:38 -0400858 if (group >= ngroups)
Alex Tomasc9de5602008-01-29 00:19:52 -0500859 break;
860
Amir Goldstein9b8b7d32011-05-09 21:49:42 -0400861 if (!bh[group - first_group])
862 /* skip initialized uptodate buddy */
863 continue;
864
Alex Tomasc9de5602008-01-29 00:19:52 -0500865 /*
866 * data carry information regarding this
867 * particular group in the format specified
868 * above
869 *
870 */
871 data = page_address(page) + (i * blocksize);
872 bitmap = bh[group - first_group]->b_data;
873
874 /*
875 * We place the buddy block and bitmap block
876 * close together
877 */
878 if ((first_block + i) & 1) {
879 /* this is block of buddy */
880 BUG_ON(incore == NULL);
Theodore Ts'o6ba495e2009-09-18 13:38:55 -0400881 mb_debug(1, "put buddy for group %u in page %lu/%x\n",
Alex Tomasc9de5602008-01-29 00:19:52 -0500882 group, page->index, i * blocksize);
Theodore Ts'of3073332010-05-17 03:00:00 -0400883 trace_ext4_mb_buddy_bitmap_load(sb, group);
Alex Tomasc9de5602008-01-29 00:19:52 -0500884 grinfo = ext4_get_group_info(sb, group);
885 grinfo->bb_fragments = 0;
886 memset(grinfo->bb_counters, 0,
Eric Sandeen19278052009-08-25 22:36:25 -0400887 sizeof(*grinfo->bb_counters) *
888 (sb->s_blocksize_bits+2));
Alex Tomasc9de5602008-01-29 00:19:52 -0500889 /*
890 * incore got set to the group block bitmap below
891 */
Aneesh Kumar K.V7a2fcbf2009-01-05 21:36:55 -0500892 ext4_lock_group(sb, group);
Amir Goldstein9b8b7d32011-05-09 21:49:42 -0400893 /* init the buddy */
894 memset(data, 0xff, blocksize);
Alex Tomasc9de5602008-01-29 00:19:52 -0500895 ext4_mb_generate_buddy(sb, data, incore, group);
Aneesh Kumar K.V7a2fcbf2009-01-05 21:36:55 -0500896 ext4_unlock_group(sb, group);
Alex Tomasc9de5602008-01-29 00:19:52 -0500897 incore = NULL;
898 } else {
899 /* this is block of bitmap */
900 BUG_ON(incore != NULL);
Theodore Ts'o6ba495e2009-09-18 13:38:55 -0400901 mb_debug(1, "put bitmap for group %u in page %lu/%x\n",
Alex Tomasc9de5602008-01-29 00:19:52 -0500902 group, page->index, i * blocksize);
Theodore Ts'of3073332010-05-17 03:00:00 -0400903 trace_ext4_mb_bitmap_load(sb, group);
Alex Tomasc9de5602008-01-29 00:19:52 -0500904
905 /* see comments in ext4_mb_put_pa() */
906 ext4_lock_group(sb, group);
907 memcpy(data, bitmap, blocksize);
908
909 /* mark all preallocated blks used in in-core bitmap */
910 ext4_mb_generate_from_pa(sb, data, group);
Aneesh Kumar K.V7a2fcbf2009-01-05 21:36:55 -0500911 ext4_mb_generate_from_freelist(sb, data, group);
Alex Tomasc9de5602008-01-29 00:19:52 -0500912 ext4_unlock_group(sb, group);
913
914 /* set incore so that the buddy information can be
915 * generated using this
916 */
917 incore = data;
918 }
919 }
920 SetPageUptodate(page);
921
922out:
923 if (bh) {
Amir Goldstein9b8b7d32011-05-09 21:49:42 -0400924 for (i = 0; i < groups_per_page; i++)
Alex Tomasc9de5602008-01-29 00:19:52 -0500925 brelse(bh[i]);
926 if (bh != &bhs)
927 kfree(bh);
928 }
929 return err;
930}
931
Curt Wohlgemuth8a57d9d2010-05-16 15:00:00 -0400932/*
Amir Goldstein2de88072011-05-09 21:48:13 -0400933 * Lock the buddy and bitmap pages. This make sure other parallel init_group
934 * on the same buddy page doesn't happen whild holding the buddy page lock.
935 * Return locked buddy and bitmap pages on e4b struct. If buddy and bitmap
936 * are on the same page e4b->bd_buddy_page is NULL and return value is 0.
Eric Sandeeneee4adc2010-10-27 21:30:15 -0400937 */
Amir Goldstein2de88072011-05-09 21:48:13 -0400938static int ext4_mb_get_buddy_page_lock(struct super_block *sb,
939 ext4_group_t group, struct ext4_buddy *e4b)
Eric Sandeeneee4adc2010-10-27 21:30:15 -0400940{
Amir Goldstein2de88072011-05-09 21:48:13 -0400941 struct inode *inode = EXT4_SB(sb)->s_buddy_cache;
942 int block, pnum, poff;
Eric Sandeeneee4adc2010-10-27 21:30:15 -0400943 int blocks_per_page;
Amir Goldstein2de88072011-05-09 21:48:13 -0400944 struct page *page;
945
946 e4b->bd_buddy_page = NULL;
947 e4b->bd_bitmap_page = NULL;
Eric Sandeeneee4adc2010-10-27 21:30:15 -0400948
949 blocks_per_page = PAGE_CACHE_SIZE / sb->s_blocksize;
950 /*
951 * the buddy cache inode stores the block bitmap
952 * and buddy information in consecutive blocks.
953 * So for each group we need two blocks.
954 */
955 block = group * 2;
956 pnum = block / blocks_per_page;
Amir Goldstein2de88072011-05-09 21:48:13 -0400957 poff = block % blocks_per_page;
958 page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS);
959 if (!page)
960 return -EIO;
961 BUG_ON(page->mapping != inode->i_mapping);
962 e4b->bd_bitmap_page = page;
963 e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize);
Eric Sandeeneee4adc2010-10-27 21:30:15 -0400964
Amir Goldstein2de88072011-05-09 21:48:13 -0400965 if (blocks_per_page >= 2) {
966 /* buddy and bitmap are on the same page */
967 return 0;
Eric Sandeeneee4adc2010-10-27 21:30:15 -0400968 }
Amir Goldstein2de88072011-05-09 21:48:13 -0400969
970 block++;
971 pnum = block / blocks_per_page;
972 poff = block % blocks_per_page;
973 page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS);
974 if (!page)
975 return -EIO;
976 BUG_ON(page->mapping != inode->i_mapping);
977 e4b->bd_buddy_page = page;
978 return 0;
Eric Sandeeneee4adc2010-10-27 21:30:15 -0400979}
980
Amir Goldstein2de88072011-05-09 21:48:13 -0400981static void ext4_mb_put_buddy_page_lock(struct ext4_buddy *e4b)
Eric Sandeeneee4adc2010-10-27 21:30:15 -0400982{
Amir Goldstein2de88072011-05-09 21:48:13 -0400983 if (e4b->bd_bitmap_page) {
984 unlock_page(e4b->bd_bitmap_page);
985 page_cache_release(e4b->bd_bitmap_page);
Eric Sandeeneee4adc2010-10-27 21:30:15 -0400986 }
Amir Goldstein2de88072011-05-09 21:48:13 -0400987 if (e4b->bd_buddy_page) {
988 unlock_page(e4b->bd_buddy_page);
989 page_cache_release(e4b->bd_buddy_page);
990 }
Eric Sandeeneee4adc2010-10-27 21:30:15 -0400991}
992
993/*
Curt Wohlgemuth8a57d9d2010-05-16 15:00:00 -0400994 * Locking note: This routine calls ext4_mb_init_cache(), which takes the
995 * block group lock of all groups for this page; do not hold the BG lock when
996 * calling this routine!
997 */
Aneesh Kumar K.Vb6a758e2009-09-09 23:47:46 -0400998static noinline_for_stack
999int ext4_mb_init_group(struct super_block *sb, ext4_group_t group)
1000{
1001
Aneesh Kumar K.Vb6a758e2009-09-09 23:47:46 -04001002 struct ext4_group_info *this_grp;
Amir Goldstein2de88072011-05-09 21:48:13 -04001003 struct ext4_buddy e4b;
1004 struct page *page;
1005 int ret = 0;
Aneesh Kumar K.Vb6a758e2009-09-09 23:47:46 -04001006
1007 mb_debug(1, "init group %u\n", group);
Aneesh Kumar K.Vb6a758e2009-09-09 23:47:46 -04001008 this_grp = ext4_get_group_info(sb, group);
1009 /*
Aneesh Kumar K.V08c3a812009-09-09 23:50:17 -04001010 * This ensures that we don't reinit the buddy cache
1011 * page which map to the group from which we are already
1012 * allocating. If we are looking at the buddy cache we would
1013 * have taken a reference using ext4_mb_load_buddy and that
Amir Goldstein2de88072011-05-09 21:48:13 -04001014 * would have pinned buddy page to page cache.
Aneesh Kumar K.Vb6a758e2009-09-09 23:47:46 -04001015 */
Amir Goldstein2de88072011-05-09 21:48:13 -04001016 ret = ext4_mb_get_buddy_page_lock(sb, group, &e4b);
1017 if (ret || !EXT4_MB_GRP_NEED_INIT(this_grp)) {
Aneesh Kumar K.Vb6a758e2009-09-09 23:47:46 -04001018 /*
1019 * somebody initialized the group
1020 * return without doing anything
1021 */
Aneesh Kumar K.Vb6a758e2009-09-09 23:47:46 -04001022 goto err;
1023 }
Amir Goldstein2de88072011-05-09 21:48:13 -04001024
1025 page = e4b.bd_bitmap_page;
1026 ret = ext4_mb_init_cache(page, NULL);
1027 if (ret)
1028 goto err;
1029 if (!PageUptodate(page)) {
Aneesh Kumar K.Vb6a758e2009-09-09 23:47:46 -04001030 ret = -EIO;
1031 goto err;
1032 }
1033 mark_page_accessed(page);
Aneesh Kumar K.Vb6a758e2009-09-09 23:47:46 -04001034
Amir Goldstein2de88072011-05-09 21:48:13 -04001035 if (e4b.bd_buddy_page == NULL) {
Aneesh Kumar K.Vb6a758e2009-09-09 23:47:46 -04001036 /*
1037 * If both the bitmap and buddy are in
1038 * the same page we don't need to force
1039 * init the buddy
1040 */
Amir Goldstein2de88072011-05-09 21:48:13 -04001041 ret = 0;
1042 goto err;
Aneesh Kumar K.Vb6a758e2009-09-09 23:47:46 -04001043 }
Amir Goldstein2de88072011-05-09 21:48:13 -04001044 /* init buddy cache */
1045 page = e4b.bd_buddy_page;
1046 ret = ext4_mb_init_cache(page, e4b.bd_bitmap);
1047 if (ret)
1048 goto err;
1049 if (!PageUptodate(page)) {
Aneesh Kumar K.Vb6a758e2009-09-09 23:47:46 -04001050 ret = -EIO;
1051 goto err;
1052 }
1053 mark_page_accessed(page);
1054err:
Amir Goldstein2de88072011-05-09 21:48:13 -04001055 ext4_mb_put_buddy_page_lock(&e4b);
Aneesh Kumar K.Vb6a758e2009-09-09 23:47:46 -04001056 return ret;
1057}
1058
Curt Wohlgemuth8a57d9d2010-05-16 15:00:00 -04001059/*
1060 * Locking note: This routine calls ext4_mb_init_cache(), which takes the
1061 * block group lock of all groups for this page; do not hold the BG lock when
1062 * calling this routine!
1063 */
Eric Sandeen4ddfef72008-04-29 08:11:12 -04001064static noinline_for_stack int
1065ext4_mb_load_buddy(struct super_block *sb, ext4_group_t group,
1066 struct ext4_buddy *e4b)
Alex Tomasc9de5602008-01-29 00:19:52 -05001067{
Alex Tomasc9de5602008-01-29 00:19:52 -05001068 int blocks_per_page;
1069 int block;
1070 int pnum;
1071 int poff;
1072 struct page *page;
Shen Fengfdf6c7a2008-07-11 19:27:31 -04001073 int ret;
Aneesh Kumar K.V920313a2009-01-05 21:36:19 -05001074 struct ext4_group_info *grp;
1075 struct ext4_sb_info *sbi = EXT4_SB(sb);
1076 struct inode *inode = sbi->s_buddy_cache;
Alex Tomasc9de5602008-01-29 00:19:52 -05001077
Theodore Ts'o6ba495e2009-09-18 13:38:55 -04001078 mb_debug(1, "load group %u\n", group);
Alex Tomasc9de5602008-01-29 00:19:52 -05001079
1080 blocks_per_page = PAGE_CACHE_SIZE / sb->s_blocksize;
Aneesh Kumar K.V920313a2009-01-05 21:36:19 -05001081 grp = ext4_get_group_info(sb, group);
Alex Tomasc9de5602008-01-29 00:19:52 -05001082
1083 e4b->bd_blkbits = sb->s_blocksize_bits;
Tao Ma529da702011-07-23 16:07:26 -04001084 e4b->bd_info = grp;
Alex Tomasc9de5602008-01-29 00:19:52 -05001085 e4b->bd_sb = sb;
1086 e4b->bd_group = group;
1087 e4b->bd_buddy_page = NULL;
1088 e4b->bd_bitmap_page = NULL;
1089
Aneesh Kumar K.Vf41c0752009-09-09 23:34:50 -04001090 if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) {
Aneesh Kumar K.Vf41c0752009-09-09 23:34:50 -04001091 /*
1092 * we need full data about the group
1093 * to make a good selection
1094 */
1095 ret = ext4_mb_init_group(sb, group);
1096 if (ret)
1097 return ret;
Aneesh Kumar K.Vf41c0752009-09-09 23:34:50 -04001098 }
1099
Alex Tomasc9de5602008-01-29 00:19:52 -05001100 /*
1101 * the buddy cache inode stores the block bitmap
1102 * and buddy information in consecutive blocks.
1103 * So for each group we need two blocks.
1104 */
1105 block = group * 2;
1106 pnum = block / blocks_per_page;
1107 poff = block % blocks_per_page;
1108
1109 /* we could use find_or_create_page(), but it locks page
1110 * what we'd like to avoid in fast path ... */
1111 page = find_get_page(inode->i_mapping, pnum);
1112 if (page == NULL || !PageUptodate(page)) {
1113 if (page)
Aneesh Kumar K.V920313a2009-01-05 21:36:19 -05001114 /*
1115 * drop the page reference and try
1116 * to get the page with lock. If we
1117 * are not uptodate that implies
1118 * somebody just created the page but
1119 * is yet to initialize the same. So
1120 * wait for it to initialize.
1121 */
Alex Tomasc9de5602008-01-29 00:19:52 -05001122 page_cache_release(page);
1123 page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS);
1124 if (page) {
1125 BUG_ON(page->mapping != inode->i_mapping);
1126 if (!PageUptodate(page)) {
Shen Fengfdf6c7a2008-07-11 19:27:31 -04001127 ret = ext4_mb_init_cache(page, NULL);
1128 if (ret) {
1129 unlock_page(page);
1130 goto err;
1131 }
Alex Tomasc9de5602008-01-29 00:19:52 -05001132 mb_cmp_bitmaps(e4b, page_address(page) +
1133 (poff * sb->s_blocksize));
1134 }
1135 unlock_page(page);
1136 }
1137 }
Shen Fengfdf6c7a2008-07-11 19:27:31 -04001138 if (page == NULL || !PageUptodate(page)) {
1139 ret = -EIO;
Alex Tomasc9de5602008-01-29 00:19:52 -05001140 goto err;
Shen Fengfdf6c7a2008-07-11 19:27:31 -04001141 }
Alex Tomasc9de5602008-01-29 00:19:52 -05001142 e4b->bd_bitmap_page = page;
1143 e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize);
1144 mark_page_accessed(page);
1145
1146 block++;
1147 pnum = block / blocks_per_page;
1148 poff = block % blocks_per_page;
1149
1150 page = find_get_page(inode->i_mapping, pnum);
1151 if (page == NULL || !PageUptodate(page)) {
1152 if (page)
1153 page_cache_release(page);
1154 page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS);
1155 if (page) {
1156 BUG_ON(page->mapping != inode->i_mapping);
Shen Fengfdf6c7a2008-07-11 19:27:31 -04001157 if (!PageUptodate(page)) {
1158 ret = ext4_mb_init_cache(page, e4b->bd_bitmap);
1159 if (ret) {
1160 unlock_page(page);
1161 goto err;
1162 }
1163 }
Alex Tomasc9de5602008-01-29 00:19:52 -05001164 unlock_page(page);
1165 }
1166 }
Shen Fengfdf6c7a2008-07-11 19:27:31 -04001167 if (page == NULL || !PageUptodate(page)) {
1168 ret = -EIO;
Alex Tomasc9de5602008-01-29 00:19:52 -05001169 goto err;
Shen Fengfdf6c7a2008-07-11 19:27:31 -04001170 }
Alex Tomasc9de5602008-01-29 00:19:52 -05001171 e4b->bd_buddy_page = page;
1172 e4b->bd_buddy = page_address(page) + (poff * sb->s_blocksize);
1173 mark_page_accessed(page);
1174
1175 BUG_ON(e4b->bd_bitmap_page == NULL);
1176 BUG_ON(e4b->bd_buddy_page == NULL);
1177
1178 return 0;
1179
1180err:
Yang Ruirui26626f112011-04-16 19:17:48 -04001181 if (page)
1182 page_cache_release(page);
Alex Tomasc9de5602008-01-29 00:19:52 -05001183 if (e4b->bd_bitmap_page)
1184 page_cache_release(e4b->bd_bitmap_page);
1185 if (e4b->bd_buddy_page)
1186 page_cache_release(e4b->bd_buddy_page);
1187 e4b->bd_buddy = NULL;
1188 e4b->bd_bitmap = NULL;
Shen Fengfdf6c7a2008-07-11 19:27:31 -04001189 return ret;
Alex Tomasc9de5602008-01-29 00:19:52 -05001190}
1191
Jing Zhange39e07f2010-05-14 00:00:00 -04001192static void ext4_mb_unload_buddy(struct ext4_buddy *e4b)
Alex Tomasc9de5602008-01-29 00:19:52 -05001193{
1194 if (e4b->bd_bitmap_page)
1195 page_cache_release(e4b->bd_bitmap_page);
1196 if (e4b->bd_buddy_page)
1197 page_cache_release(e4b->bd_buddy_page);
1198}
1199
1200
1201static int mb_find_order_for_block(struct ext4_buddy *e4b, int block)
1202{
1203 int order = 1;
1204 void *bb;
1205
Theodore Ts'oc5e8f3f2012-02-20 17:54:06 -05001206 BUG_ON(e4b->bd_bitmap == e4b->bd_buddy);
Alex Tomasc9de5602008-01-29 00:19:52 -05001207 BUG_ON(block >= (1 << (e4b->bd_blkbits + 3)));
1208
Theodore Ts'oc5e8f3f2012-02-20 17:54:06 -05001209 bb = e4b->bd_buddy;
Alex Tomasc9de5602008-01-29 00:19:52 -05001210 while (order <= e4b->bd_blkbits + 1) {
1211 block = block >> 1;
1212 if (!mb_test_bit(block, bb)) {
1213 /* this block is part of buddy of order 'order' */
1214 return order;
1215 }
1216 bb += 1 << (e4b->bd_blkbits - order);
1217 order++;
1218 }
1219 return 0;
1220}
1221
Aneesh Kumar K.V955ce5f2009-05-02 20:35:09 -04001222static void mb_clear_bits(void *bm, int cur, int len)
Alex Tomasc9de5602008-01-29 00:19:52 -05001223{
1224 __u32 *addr;
1225
1226 len = cur + len;
1227 while (cur < len) {
1228 if ((cur & 31) == 0 && (len - cur) >= 32) {
1229 /* fast path: clear whole word at once */
1230 addr = bm + (cur >> 3);
1231 *addr = 0;
1232 cur += 32;
1233 continue;
1234 }
Aneesh Kumar K.V955ce5f2009-05-02 20:35:09 -04001235 mb_clear_bit(cur, bm);
Alex Tomasc9de5602008-01-29 00:19:52 -05001236 cur++;
1237 }
1238}
1239
Yongqiang Yangc3e94d12011-07-26 22:05:53 -04001240void ext4_set_bits(void *bm, int cur, int len)
Alex Tomasc9de5602008-01-29 00:19:52 -05001241{
1242 __u32 *addr;
1243
1244 len = cur + len;
1245 while (cur < len) {
1246 if ((cur & 31) == 0 && (len - cur) >= 32) {
1247 /* fast path: set whole word at once */
1248 addr = bm + (cur >> 3);
1249 *addr = 0xffffffff;
1250 cur += 32;
1251 continue;
1252 }
Aneesh Kumar K.V955ce5f2009-05-02 20:35:09 -04001253 mb_set_bit(cur, bm);
Alex Tomasc9de5602008-01-29 00:19:52 -05001254 cur++;
1255 }
1256}
1257
Shen Feng7e5a8cd2008-07-13 21:03:31 -04001258static void mb_free_blocks(struct inode *inode, struct ext4_buddy *e4b,
Alex Tomasc9de5602008-01-29 00:19:52 -05001259 int first, int count)
1260{
1261 int block = 0;
1262 int max = 0;
1263 int order;
1264 void *buddy;
1265 void *buddy2;
1266 struct super_block *sb = e4b->bd_sb;
1267
1268 BUG_ON(first + count > (sb->s_blocksize << 3));
Vincent Minetbc8e6742009-05-15 08:33:18 -04001269 assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group));
Alex Tomasc9de5602008-01-29 00:19:52 -05001270 mb_check_buddy(e4b);
1271 mb_free_blocks_double(inode, e4b, first, count);
1272
1273 e4b->bd_info->bb_free += count;
1274 if (first < e4b->bd_info->bb_first_free)
1275 e4b->bd_info->bb_first_free = first;
1276
1277 /* let's maintain fragments counter */
1278 if (first != 0)
Theodore Ts'oc5e8f3f2012-02-20 17:54:06 -05001279 block = !mb_test_bit(first - 1, e4b->bd_bitmap);
Alex Tomasc9de5602008-01-29 00:19:52 -05001280 if (first + count < EXT4_SB(sb)->s_mb_maxs[0])
Theodore Ts'oc5e8f3f2012-02-20 17:54:06 -05001281 max = !mb_test_bit(first + count, e4b->bd_bitmap);
Alex Tomasc9de5602008-01-29 00:19:52 -05001282 if (block && max)
1283 e4b->bd_info->bb_fragments--;
1284 else if (!block && !max)
1285 e4b->bd_info->bb_fragments++;
1286
1287 /* let's maintain buddy itself */
1288 while (count-- > 0) {
1289 block = first++;
1290 order = 0;
1291
Theodore Ts'oc5e8f3f2012-02-20 17:54:06 -05001292 if (!mb_test_bit(block, e4b->bd_bitmap)) {
Alex Tomasc9de5602008-01-29 00:19:52 -05001293 ext4_fsblk_t blocknr;
Akinobu Mita5661bd62010-03-03 23:53:39 -05001294
1295 blocknr = ext4_group_first_block_no(sb, e4b->bd_group);
Theodore Ts'o53accfa2011-09-09 18:48:51 -04001296 blocknr += EXT4_C2B(EXT4_SB(sb), block);
Aneesh Kumar K.V5d1b1b32009-01-05 22:19:52 -05001297 ext4_grp_locked_error(sb, e4b->bd_group,
Theodore Ts'oe29136f2010-06-29 12:54:28 -04001298 inode ? inode->i_ino : 0,
1299 blocknr,
1300 "freeing already freed block "
1301 "(bit %u)", block);
Alex Tomasc9de5602008-01-29 00:19:52 -05001302 }
Theodore Ts'oc5e8f3f2012-02-20 17:54:06 -05001303 mb_clear_bit(block, e4b->bd_bitmap);
Alex Tomasc9de5602008-01-29 00:19:52 -05001304 e4b->bd_info->bb_counters[order]++;
1305
1306 /* start of the buddy */
1307 buddy = mb_find_buddy(e4b, order, &max);
1308
1309 do {
1310 block &= ~1UL;
1311 if (mb_test_bit(block, buddy) ||
1312 mb_test_bit(block + 1, buddy))
1313 break;
1314
1315 /* both the buddies are free, try to coalesce them */
1316 buddy2 = mb_find_buddy(e4b, order + 1, &max);
1317
1318 if (!buddy2)
1319 break;
1320
1321 if (order > 0) {
1322 /* for special purposes, we don't set
1323 * free bits in bitmap */
1324 mb_set_bit(block, buddy);
1325 mb_set_bit(block + 1, buddy);
1326 }
1327 e4b->bd_info->bb_counters[order]--;
1328 e4b->bd_info->bb_counters[order]--;
1329
1330 block = block >> 1;
1331 order++;
1332 e4b->bd_info->bb_counters[order]++;
1333
1334 mb_clear_bit(block, buddy2);
1335 buddy = buddy2;
1336 } while (1);
1337 }
Curt Wohlgemuth8a57d9d2010-05-16 15:00:00 -04001338 mb_set_largest_free_order(sb, e4b->bd_info);
Alex Tomasc9de5602008-01-29 00:19:52 -05001339 mb_check_buddy(e4b);
Alex Tomasc9de5602008-01-29 00:19:52 -05001340}
1341
1342static int mb_find_extent(struct ext4_buddy *e4b, int order, int block,
1343 int needed, struct ext4_free_extent *ex)
1344{
1345 int next = block;
1346 int max;
Alex Tomasc9de5602008-01-29 00:19:52 -05001347 void *buddy;
1348
Vincent Minetbc8e6742009-05-15 08:33:18 -04001349 assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));
Alex Tomasc9de5602008-01-29 00:19:52 -05001350 BUG_ON(ex == NULL);
1351
1352 buddy = mb_find_buddy(e4b, order, &max);
1353 BUG_ON(buddy == NULL);
1354 BUG_ON(block >= max);
1355 if (mb_test_bit(block, buddy)) {
1356 ex->fe_len = 0;
1357 ex->fe_start = 0;
1358 ex->fe_group = 0;
1359 return 0;
1360 }
1361
1362 /* FIXME dorp order completely ? */
1363 if (likely(order == 0)) {
1364 /* find actual order */
1365 order = mb_find_order_for_block(e4b, block);
1366 block = block >> order;
1367 }
1368
1369 ex->fe_len = 1 << order;
1370 ex->fe_start = block << order;
1371 ex->fe_group = e4b->bd_group;
1372
1373 /* calc difference from given start */
1374 next = next - ex->fe_start;
1375 ex->fe_len -= next;
1376 ex->fe_start += next;
1377
1378 while (needed > ex->fe_len &&
1379 (buddy = mb_find_buddy(e4b, order, &max))) {
1380
1381 if (block + 1 >= max)
1382 break;
1383
1384 next = (block + 1) * (1 << order);
Theodore Ts'oc5e8f3f2012-02-20 17:54:06 -05001385 if (mb_test_bit(next, e4b->bd_bitmap))
Alex Tomasc9de5602008-01-29 00:19:52 -05001386 break;
1387
Robin Dongb051d8d2011-10-26 05:30:30 -04001388 order = mb_find_order_for_block(e4b, next);
Alex Tomasc9de5602008-01-29 00:19:52 -05001389
Alex Tomasc9de5602008-01-29 00:19:52 -05001390 block = next >> order;
1391 ex->fe_len += 1 << order;
1392 }
1393
1394 BUG_ON(ex->fe_start + ex->fe_len > (1 << (e4b->bd_blkbits + 3)));
1395 return ex->fe_len;
1396}
1397
1398static int mb_mark_used(struct ext4_buddy *e4b, struct ext4_free_extent *ex)
1399{
1400 int ord;
1401 int mlen = 0;
1402 int max = 0;
1403 int cur;
1404 int start = ex->fe_start;
1405 int len = ex->fe_len;
1406 unsigned ret = 0;
1407 int len0 = len;
1408 void *buddy;
1409
1410 BUG_ON(start + len > (e4b->bd_sb->s_blocksize << 3));
1411 BUG_ON(e4b->bd_group != ex->fe_group);
Vincent Minetbc8e6742009-05-15 08:33:18 -04001412 assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));
Alex Tomasc9de5602008-01-29 00:19:52 -05001413 mb_check_buddy(e4b);
1414 mb_mark_used_double(e4b, start, len);
1415
1416 e4b->bd_info->bb_free -= len;
1417 if (e4b->bd_info->bb_first_free == start)
1418 e4b->bd_info->bb_first_free += len;
1419
1420 /* let's maintain fragments counter */
1421 if (start != 0)
Theodore Ts'oc5e8f3f2012-02-20 17:54:06 -05001422 mlen = !mb_test_bit(start - 1, e4b->bd_bitmap);
Alex Tomasc9de5602008-01-29 00:19:52 -05001423 if (start + len < EXT4_SB(e4b->bd_sb)->s_mb_maxs[0])
Theodore Ts'oc5e8f3f2012-02-20 17:54:06 -05001424 max = !mb_test_bit(start + len, e4b->bd_bitmap);
Alex Tomasc9de5602008-01-29 00:19:52 -05001425 if (mlen && max)
1426 e4b->bd_info->bb_fragments++;
1427 else if (!mlen && !max)
1428 e4b->bd_info->bb_fragments--;
1429
1430 /* let's maintain buddy itself */
1431 while (len) {
1432 ord = mb_find_order_for_block(e4b, start);
1433
1434 if (((start >> ord) << ord) == start && len >= (1 << ord)) {
1435 /* the whole chunk may be allocated at once! */
1436 mlen = 1 << ord;
1437 buddy = mb_find_buddy(e4b, ord, &max);
1438 BUG_ON((start >> ord) >= max);
1439 mb_set_bit(start >> ord, buddy);
1440 e4b->bd_info->bb_counters[ord]--;
1441 start += mlen;
1442 len -= mlen;
1443 BUG_ON(len < 0);
1444 continue;
1445 }
1446
1447 /* store for history */
1448 if (ret == 0)
1449 ret = len | (ord << 16);
1450
1451 /* we have to split large buddy */
1452 BUG_ON(ord <= 0);
1453 buddy = mb_find_buddy(e4b, ord, &max);
1454 mb_set_bit(start >> ord, buddy);
1455 e4b->bd_info->bb_counters[ord]--;
1456
1457 ord--;
1458 cur = (start >> ord) & ~1U;
1459 buddy = mb_find_buddy(e4b, ord, &max);
1460 mb_clear_bit(cur, buddy);
1461 mb_clear_bit(cur + 1, buddy);
1462 e4b->bd_info->bb_counters[ord]++;
1463 e4b->bd_info->bb_counters[ord]++;
1464 }
Curt Wohlgemuth8a57d9d2010-05-16 15:00:00 -04001465 mb_set_largest_free_order(e4b->bd_sb, e4b->bd_info);
Alex Tomasc9de5602008-01-29 00:19:52 -05001466
Theodore Ts'oc5e8f3f2012-02-20 17:54:06 -05001467 ext4_set_bits(e4b->bd_bitmap, ex->fe_start, len0);
Alex Tomasc9de5602008-01-29 00:19:52 -05001468 mb_check_buddy(e4b);
1469
1470 return ret;
1471}
1472
1473/*
1474 * Must be called under group lock!
1475 */
1476static void ext4_mb_use_best_found(struct ext4_allocation_context *ac,
1477 struct ext4_buddy *e4b)
1478{
1479 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
1480 int ret;
1481
1482 BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group);
1483 BUG_ON(ac->ac_status == AC_STATUS_FOUND);
1484
1485 ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len);
1486 ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical;
1487 ret = mb_mark_used(e4b, &ac->ac_b_ex);
1488
1489 /* preallocation can change ac_b_ex, thus we store actually
1490 * allocated blocks for history */
1491 ac->ac_f_ex = ac->ac_b_ex;
1492
1493 ac->ac_status = AC_STATUS_FOUND;
1494 ac->ac_tail = ret & 0xffff;
1495 ac->ac_buddy = ret >> 16;
1496
Aneesh Kumar K.Vc3a326a2008-11-25 15:11:52 -05001497 /*
1498 * take the page reference. We want the page to be pinned
1499 * so that we don't get a ext4_mb_init_cache_call for this
1500 * group until we update the bitmap. That would mean we
1501 * double allocate blocks. The reference is dropped
1502 * in ext4_mb_release_context
1503 */
Alex Tomasc9de5602008-01-29 00:19:52 -05001504 ac->ac_bitmap_page = e4b->bd_bitmap_page;
1505 get_page(ac->ac_bitmap_page);
1506 ac->ac_buddy_page = e4b->bd_buddy_page;
1507 get_page(ac->ac_buddy_page);
Alex Tomasc9de5602008-01-29 00:19:52 -05001508 /* store last allocated for subsequent stream allocation */
Theodore Ts'o4ba74d02009-08-09 22:01:13 -04001509 if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) {
Alex Tomasc9de5602008-01-29 00:19:52 -05001510 spin_lock(&sbi->s_md_lock);
1511 sbi->s_mb_last_group = ac->ac_f_ex.fe_group;
1512 sbi->s_mb_last_start = ac->ac_f_ex.fe_start;
1513 spin_unlock(&sbi->s_md_lock);
1514 }
1515}
1516
1517/*
1518 * regular allocator, for general purposes allocation
1519 */
1520
1521static void ext4_mb_check_limits(struct ext4_allocation_context *ac,
1522 struct ext4_buddy *e4b,
1523 int finish_group)
1524{
1525 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
1526 struct ext4_free_extent *bex = &ac->ac_b_ex;
1527 struct ext4_free_extent *gex = &ac->ac_g_ex;
1528 struct ext4_free_extent ex;
1529 int max;
1530
Aneesh Kumar K.V032115f2009-01-05 21:34:30 -05001531 if (ac->ac_status == AC_STATUS_FOUND)
1532 return;
Alex Tomasc9de5602008-01-29 00:19:52 -05001533 /*
1534 * We don't want to scan for a whole year
1535 */
1536 if (ac->ac_found > sbi->s_mb_max_to_scan &&
1537 !(ac->ac_flags & EXT4_MB_HINT_FIRST)) {
1538 ac->ac_status = AC_STATUS_BREAK;
1539 return;
1540 }
1541
1542 /*
1543 * Haven't found good chunk so far, let's continue
1544 */
1545 if (bex->fe_len < gex->fe_len)
1546 return;
1547
1548 if ((finish_group || ac->ac_found > sbi->s_mb_min_to_scan)
1549 && bex->fe_group == e4b->bd_group) {
1550 /* recheck chunk's availability - we don't know
1551 * when it was found (within this lock-unlock
1552 * period or not) */
1553 max = mb_find_extent(e4b, 0, bex->fe_start, gex->fe_len, &ex);
1554 if (max >= gex->fe_len) {
1555 ext4_mb_use_best_found(ac, e4b);
1556 return;
1557 }
1558 }
1559}
1560
1561/*
1562 * The routine checks whether found extent is good enough. If it is,
1563 * then the extent gets marked used and flag is set to the context
1564 * to stop scanning. Otherwise, the extent is compared with the
1565 * previous found extent and if new one is better, then it's stored
1566 * in the context. Later, the best found extent will be used, if
1567 * mballoc can't find good enough extent.
1568 *
1569 * FIXME: real allocation policy is to be designed yet!
1570 */
1571static void ext4_mb_measure_extent(struct ext4_allocation_context *ac,
1572 struct ext4_free_extent *ex,
1573 struct ext4_buddy *e4b)
1574{
1575 struct ext4_free_extent *bex = &ac->ac_b_ex;
1576 struct ext4_free_extent *gex = &ac->ac_g_ex;
1577
1578 BUG_ON(ex->fe_len <= 0);
Theodore Ts'o7137d7a2011-09-09 18:38:51 -04001579 BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb));
1580 BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb));
Alex Tomasc9de5602008-01-29 00:19:52 -05001581 BUG_ON(ac->ac_status != AC_STATUS_CONTINUE);
1582
1583 ac->ac_found++;
1584
1585 /*
1586 * The special case - take what you catch first
1587 */
1588 if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) {
1589 *bex = *ex;
1590 ext4_mb_use_best_found(ac, e4b);
1591 return;
1592 }
1593
1594 /*
1595 * Let's check whether the chuck is good enough
1596 */
1597 if (ex->fe_len == gex->fe_len) {
1598 *bex = *ex;
1599 ext4_mb_use_best_found(ac, e4b);
1600 return;
1601 }
1602
1603 /*
1604 * If this is first found extent, just store it in the context
1605 */
1606 if (bex->fe_len == 0) {
1607 *bex = *ex;
1608 return;
1609 }
1610
1611 /*
1612 * If new found extent is better, store it in the context
1613 */
1614 if (bex->fe_len < gex->fe_len) {
1615 /* if the request isn't satisfied, any found extent
1616 * larger than previous best one is better */
1617 if (ex->fe_len > bex->fe_len)
1618 *bex = *ex;
1619 } else if (ex->fe_len > gex->fe_len) {
1620 /* if the request is satisfied, then we try to find
1621 * an extent that still satisfy the request, but is
1622 * smaller than previous one */
1623 if (ex->fe_len < bex->fe_len)
1624 *bex = *ex;
1625 }
1626
1627 ext4_mb_check_limits(ac, e4b, 0);
1628}
1629
Eric Sandeen089ceec2009-07-05 22:17:31 -04001630static noinline_for_stack
1631int ext4_mb_try_best_found(struct ext4_allocation_context *ac,
Alex Tomasc9de5602008-01-29 00:19:52 -05001632 struct ext4_buddy *e4b)
1633{
1634 struct ext4_free_extent ex = ac->ac_b_ex;
1635 ext4_group_t group = ex.fe_group;
1636 int max;
1637 int err;
1638
1639 BUG_ON(ex.fe_len <= 0);
1640 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);
1641 if (err)
1642 return err;
1643
1644 ext4_lock_group(ac->ac_sb, group);
1645 max = mb_find_extent(e4b, 0, ex.fe_start, ex.fe_len, &ex);
1646
1647 if (max > 0) {
1648 ac->ac_b_ex = ex;
1649 ext4_mb_use_best_found(ac, e4b);
1650 }
1651
1652 ext4_unlock_group(ac->ac_sb, group);
Jing Zhange39e07f2010-05-14 00:00:00 -04001653 ext4_mb_unload_buddy(e4b);
Alex Tomasc9de5602008-01-29 00:19:52 -05001654
1655 return 0;
1656}
1657
Eric Sandeen089ceec2009-07-05 22:17:31 -04001658static noinline_for_stack
1659int ext4_mb_find_by_goal(struct ext4_allocation_context *ac,
Alex Tomasc9de5602008-01-29 00:19:52 -05001660 struct ext4_buddy *e4b)
1661{
1662 ext4_group_t group = ac->ac_g_ex.fe_group;
1663 int max;
1664 int err;
1665 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
Alex Tomasc9de5602008-01-29 00:19:52 -05001666 struct ext4_free_extent ex;
1667
1668 if (!(ac->ac_flags & EXT4_MB_HINT_TRY_GOAL))
1669 return 0;
1670
1671 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);
1672 if (err)
1673 return err;
1674
1675 ext4_lock_group(ac->ac_sb, group);
1676 max = mb_find_extent(e4b, 0, ac->ac_g_ex.fe_start,
1677 ac->ac_g_ex.fe_len, &ex);
1678
1679 if (max >= ac->ac_g_ex.fe_len && ac->ac_g_ex.fe_len == sbi->s_stripe) {
1680 ext4_fsblk_t start;
1681
Akinobu Mita5661bd62010-03-03 23:53:39 -05001682 start = ext4_group_first_block_no(ac->ac_sb, e4b->bd_group) +
1683 ex.fe_start;
Alex Tomasc9de5602008-01-29 00:19:52 -05001684 /* use do_div to get remainder (would be 64-bit modulo) */
1685 if (do_div(start, sbi->s_stripe) == 0) {
1686 ac->ac_found++;
1687 ac->ac_b_ex = ex;
1688 ext4_mb_use_best_found(ac, e4b);
1689 }
1690 } else if (max >= ac->ac_g_ex.fe_len) {
1691 BUG_ON(ex.fe_len <= 0);
1692 BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group);
1693 BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start);
1694 ac->ac_found++;
1695 ac->ac_b_ex = ex;
1696 ext4_mb_use_best_found(ac, e4b);
1697 } else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) {
1698 /* Sometimes, caller may want to merge even small
1699 * number of blocks to an existing extent */
1700 BUG_ON(ex.fe_len <= 0);
1701 BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group);
1702 BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start);
1703 ac->ac_found++;
1704 ac->ac_b_ex = ex;
1705 ext4_mb_use_best_found(ac, e4b);
1706 }
1707 ext4_unlock_group(ac->ac_sb, group);
Jing Zhange39e07f2010-05-14 00:00:00 -04001708 ext4_mb_unload_buddy(e4b);
Alex Tomasc9de5602008-01-29 00:19:52 -05001709
1710 return 0;
1711}
1712
1713/*
1714 * The routine scans buddy structures (not bitmap!) from given order
1715 * to max order and tries to find big enough chunk to satisfy the req
1716 */
Eric Sandeen089ceec2009-07-05 22:17:31 -04001717static noinline_for_stack
1718void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac,
Alex Tomasc9de5602008-01-29 00:19:52 -05001719 struct ext4_buddy *e4b)
1720{
1721 struct super_block *sb = ac->ac_sb;
1722 struct ext4_group_info *grp = e4b->bd_info;
1723 void *buddy;
1724 int i;
1725 int k;
1726 int max;
1727
1728 BUG_ON(ac->ac_2order <= 0);
1729 for (i = ac->ac_2order; i <= sb->s_blocksize_bits + 1; i++) {
1730 if (grp->bb_counters[i] == 0)
1731 continue;
1732
1733 buddy = mb_find_buddy(e4b, i, &max);
1734 BUG_ON(buddy == NULL);
1735
Aneesh Kumar K.Vffad0a42008-02-23 01:38:34 -05001736 k = mb_find_next_zero_bit(buddy, max, 0);
Alex Tomasc9de5602008-01-29 00:19:52 -05001737 BUG_ON(k >= max);
1738
1739 ac->ac_found++;
1740
1741 ac->ac_b_ex.fe_len = 1 << i;
1742 ac->ac_b_ex.fe_start = k << i;
1743 ac->ac_b_ex.fe_group = e4b->bd_group;
1744
1745 ext4_mb_use_best_found(ac, e4b);
1746
1747 BUG_ON(ac->ac_b_ex.fe_len != ac->ac_g_ex.fe_len);
1748
1749 if (EXT4_SB(sb)->s_mb_stats)
1750 atomic_inc(&EXT4_SB(sb)->s_bal_2orders);
1751
1752 break;
1753 }
1754}
1755
1756/*
1757 * The routine scans the group and measures all found extents.
1758 * In order to optimize scanning, caller must pass number of
1759 * free blocks in the group, so the routine can know upper limit.
1760 */
Eric Sandeen089ceec2009-07-05 22:17:31 -04001761static noinline_for_stack
1762void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac,
Alex Tomasc9de5602008-01-29 00:19:52 -05001763 struct ext4_buddy *e4b)
1764{
1765 struct super_block *sb = ac->ac_sb;
Theodore Ts'oc5e8f3f2012-02-20 17:54:06 -05001766 void *bitmap = e4b->bd_bitmap;
Alex Tomasc9de5602008-01-29 00:19:52 -05001767 struct ext4_free_extent ex;
1768 int i;
1769 int free;
1770
1771 free = e4b->bd_info->bb_free;
1772 BUG_ON(free <= 0);
1773
1774 i = e4b->bd_info->bb_first_free;
1775
1776 while (free && ac->ac_status == AC_STATUS_CONTINUE) {
Aneesh Kumar K.Vffad0a42008-02-23 01:38:34 -05001777 i = mb_find_next_zero_bit(bitmap,
Theodore Ts'o7137d7a2011-09-09 18:38:51 -04001778 EXT4_CLUSTERS_PER_GROUP(sb), i);
1779 if (i >= EXT4_CLUSTERS_PER_GROUP(sb)) {
Aneesh Kumar K.V26346ff2008-02-10 01:10:04 -05001780 /*
Aneesh Kumar K.Ve56eb652008-02-15 13:48:21 -05001781 * IF we have corrupt bitmap, we won't find any
Aneesh Kumar K.V26346ff2008-02-10 01:10:04 -05001782 * free blocks even though group info says we
1783 * we have free blocks
1784 */
Theodore Ts'oe29136f2010-06-29 12:54:28 -04001785 ext4_grp_locked_error(sb, e4b->bd_group, 0, 0,
Theodore Ts'o53accfa2011-09-09 18:48:51 -04001786 "%d free clusters as per "
Theodore Ts'ofde4d952009-01-05 22:17:35 -05001787 "group info. But bitmap says 0",
Aneesh Kumar K.V26346ff2008-02-10 01:10:04 -05001788 free);
Alex Tomasc9de5602008-01-29 00:19:52 -05001789 break;
1790 }
1791
1792 mb_find_extent(e4b, 0, i, ac->ac_g_ex.fe_len, &ex);
1793 BUG_ON(ex.fe_len <= 0);
Aneesh Kumar K.V26346ff2008-02-10 01:10:04 -05001794 if (free < ex.fe_len) {
Theodore Ts'oe29136f2010-06-29 12:54:28 -04001795 ext4_grp_locked_error(sb, e4b->bd_group, 0, 0,
Theodore Ts'o53accfa2011-09-09 18:48:51 -04001796 "%d free clusters as per "
Theodore Ts'ofde4d952009-01-05 22:17:35 -05001797 "group info. But got %d blocks",
Aneesh Kumar K.V26346ff2008-02-10 01:10:04 -05001798 free, ex.fe_len);
Aneesh Kumar K.Ve56eb652008-02-15 13:48:21 -05001799 /*
1800 * The number of free blocks differs. This mostly
1801 * indicate that the bitmap is corrupt. So exit
1802 * without claiming the space.
1803 */
1804 break;
Aneesh Kumar K.V26346ff2008-02-10 01:10:04 -05001805 }
Alex Tomasc9de5602008-01-29 00:19:52 -05001806
1807 ext4_mb_measure_extent(ac, &ex, e4b);
1808
1809 i += ex.fe_len;
1810 free -= ex.fe_len;
1811 }
1812
1813 ext4_mb_check_limits(ac, e4b, 1);
1814}
1815
1816/*
1817 * This is a special case for storages like raid5
Eric Sandeen506bf2d2010-07-27 11:56:06 -04001818 * we try to find stripe-aligned chunks for stripe-size-multiple requests
Alex Tomasc9de5602008-01-29 00:19:52 -05001819 */
Eric Sandeen089ceec2009-07-05 22:17:31 -04001820static noinline_for_stack
1821void ext4_mb_scan_aligned(struct ext4_allocation_context *ac,
Alex Tomasc9de5602008-01-29 00:19:52 -05001822 struct ext4_buddy *e4b)
1823{
1824 struct super_block *sb = ac->ac_sb;
1825 struct ext4_sb_info *sbi = EXT4_SB(sb);
Theodore Ts'oc5e8f3f2012-02-20 17:54:06 -05001826 void *bitmap = e4b->bd_bitmap;
Alex Tomasc9de5602008-01-29 00:19:52 -05001827 struct ext4_free_extent ex;
1828 ext4_fsblk_t first_group_block;
1829 ext4_fsblk_t a;
1830 ext4_grpblk_t i;
1831 int max;
1832
1833 BUG_ON(sbi->s_stripe == 0);
1834
1835 /* find first stripe-aligned block in group */
Akinobu Mita5661bd62010-03-03 23:53:39 -05001836 first_group_block = ext4_group_first_block_no(sb, e4b->bd_group);
1837
Alex Tomasc9de5602008-01-29 00:19:52 -05001838 a = first_group_block + sbi->s_stripe - 1;
1839 do_div(a, sbi->s_stripe);
1840 i = (a * sbi->s_stripe) - first_group_block;
1841
Theodore Ts'o7137d7a2011-09-09 18:38:51 -04001842 while (i < EXT4_CLUSTERS_PER_GROUP(sb)) {
Alex Tomasc9de5602008-01-29 00:19:52 -05001843 if (!mb_test_bit(i, bitmap)) {
1844 max = mb_find_extent(e4b, 0, i, sbi->s_stripe, &ex);
1845 if (max >= sbi->s_stripe) {
1846 ac->ac_found++;
1847 ac->ac_b_ex = ex;
1848 ext4_mb_use_best_found(ac, e4b);
1849 break;
1850 }
1851 }
1852 i += sbi->s_stripe;
1853 }
1854}
1855
Curt Wohlgemuth8a57d9d2010-05-16 15:00:00 -04001856/* This is now called BEFORE we load the buddy bitmap. */
Alex Tomasc9de5602008-01-29 00:19:52 -05001857static int ext4_mb_good_group(struct ext4_allocation_context *ac,
1858 ext4_group_t group, int cr)
1859{
1860 unsigned free, fragments;
Theodore Ts'oa4912122009-03-12 12:18:34 -04001861 int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb));
Alex Tomasc9de5602008-01-29 00:19:52 -05001862 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);
1863
1864 BUG_ON(cr < 0 || cr >= 4);
Curt Wohlgemuth8a57d9d2010-05-16 15:00:00 -04001865
1866 /* We only do this if the grp has never been initialized */
1867 if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) {
1868 int ret = ext4_mb_init_group(ac->ac_sb, group);
1869 if (ret)
1870 return 0;
1871 }
Alex Tomasc9de5602008-01-29 00:19:52 -05001872
1873 free = grp->bb_free;
1874 fragments = grp->bb_fragments;
1875 if (free == 0)
1876 return 0;
1877 if (fragments == 0)
1878 return 0;
1879
1880 switch (cr) {
1881 case 0:
1882 BUG_ON(ac->ac_2order == 0);
Alex Tomasc9de5602008-01-29 00:19:52 -05001883
Curt Wohlgemuth8a57d9d2010-05-16 15:00:00 -04001884 if (grp->bb_largest_free_order < ac->ac_2order)
1885 return 0;
1886
Theodore Ts'oa4912122009-03-12 12:18:34 -04001887 /* Avoid using the first bg of a flexgroup for data files */
1888 if ((ac->ac_flags & EXT4_MB_HINT_DATA) &&
1889 (flex_size >= EXT4_FLEX_SIZE_DIR_ALLOC_SCHEME) &&
1890 ((group % flex_size) == 0))
1891 return 0;
1892
Curt Wohlgemuth8a57d9d2010-05-16 15:00:00 -04001893 return 1;
Alex Tomasc9de5602008-01-29 00:19:52 -05001894 case 1:
1895 if ((free / fragments) >= ac->ac_g_ex.fe_len)
1896 return 1;
1897 break;
1898 case 2:
1899 if (free >= ac->ac_g_ex.fe_len)
1900 return 1;
1901 break;
1902 case 3:
1903 return 1;
1904 default:
1905 BUG();
1906 }
1907
1908 return 0;
1909}
1910
Eric Sandeen4ddfef72008-04-29 08:11:12 -04001911static noinline_for_stack int
1912ext4_mb_regular_allocator(struct ext4_allocation_context *ac)
Alex Tomasc9de5602008-01-29 00:19:52 -05001913{
Theodore Ts'o8df96752009-05-01 08:50:38 -04001914 ext4_group_t ngroups, group, i;
Alex Tomasc9de5602008-01-29 00:19:52 -05001915 int cr;
1916 int err = 0;
Alex Tomasc9de5602008-01-29 00:19:52 -05001917 struct ext4_sb_info *sbi;
1918 struct super_block *sb;
1919 struct ext4_buddy e4b;
Alex Tomasc9de5602008-01-29 00:19:52 -05001920
1921 sb = ac->ac_sb;
1922 sbi = EXT4_SB(sb);
Theodore Ts'o8df96752009-05-01 08:50:38 -04001923 ngroups = ext4_get_groups_count(sb);
Eric Sandeenfb0a3872009-09-16 14:45:10 -04001924 /* non-extent files are limited to low blocks/groups */
Dmitry Monakhov12e9b892010-05-16 22:00:00 -04001925 if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)))
Eric Sandeenfb0a3872009-09-16 14:45:10 -04001926 ngroups = sbi->s_blockfile_groups;
1927
Alex Tomasc9de5602008-01-29 00:19:52 -05001928 BUG_ON(ac->ac_status == AC_STATUS_FOUND);
1929
1930 /* first, try the goal */
1931 err = ext4_mb_find_by_goal(ac, &e4b);
1932 if (err || ac->ac_status == AC_STATUS_FOUND)
1933 goto out;
1934
1935 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))
1936 goto out;
1937
1938 /*
1939 * ac->ac2_order is set only if the fe_len is a power of 2
1940 * if ac2_order is set we also set criteria to 0 so that we
1941 * try exact allocation using buddy.
1942 */
1943 i = fls(ac->ac_g_ex.fe_len);
1944 ac->ac_2order = 0;
1945 /*
1946 * We search using buddy data only if the order of the request
1947 * is greater than equal to the sbi_s_mb_order2_reqs
Theodore Ts'ob713a5e2009-03-31 09:11:14 -04001948 * You can tune it via /sys/fs/ext4/<partition>/mb_order2_req
Alex Tomasc9de5602008-01-29 00:19:52 -05001949 */
1950 if (i >= sbi->s_mb_order2_reqs) {
1951 /*
1952 * This should tell if fe_len is exactly power of 2
1953 */
1954 if ((ac->ac_g_ex.fe_len & (~(1 << (i - 1)))) == 0)
1955 ac->ac_2order = i - 1;
1956 }
1957
Theodore Ts'o4ba74d02009-08-09 22:01:13 -04001958 /* if stream allocation is enabled, use global goal */
1959 if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) {
Alex Tomasc9de5602008-01-29 00:19:52 -05001960 /* TBD: may be hot point */
1961 spin_lock(&sbi->s_md_lock);
1962 ac->ac_g_ex.fe_group = sbi->s_mb_last_group;
1963 ac->ac_g_ex.fe_start = sbi->s_mb_last_start;
1964 spin_unlock(&sbi->s_md_lock);
1965 }
Theodore Ts'o4ba74d02009-08-09 22:01:13 -04001966
Alex Tomasc9de5602008-01-29 00:19:52 -05001967 /* Let's just scan groups to find more-less suitable blocks */
1968 cr = ac->ac_2order ? 0 : 1;
1969 /*
1970 * cr == 0 try to get exact allocation,
1971 * cr == 3 try to get anything
1972 */
1973repeat:
1974 for (; cr < 4 && ac->ac_status == AC_STATUS_CONTINUE; cr++) {
1975 ac->ac_criteria = cr;
Aneesh Kumar K.Ved8f9c72008-07-11 19:27:31 -04001976 /*
1977 * searching for the right group start
1978 * from the goal value specified
1979 */
1980 group = ac->ac_g_ex.fe_group;
1981
Theodore Ts'o8df96752009-05-01 08:50:38 -04001982 for (i = 0; i < ngroups; group++, i++) {
Theodore Ts'o8df96752009-05-01 08:50:38 -04001983 if (group == ngroups)
Alex Tomasc9de5602008-01-29 00:19:52 -05001984 group = 0;
1985
Curt Wohlgemuth8a57d9d2010-05-16 15:00:00 -04001986 /* This now checks without needing the buddy page */
1987 if (!ext4_mb_good_group(ac, group, cr))
Alex Tomasc9de5602008-01-29 00:19:52 -05001988 continue;
1989
Alex Tomasc9de5602008-01-29 00:19:52 -05001990 err = ext4_mb_load_buddy(sb, group, &e4b);
1991 if (err)
1992 goto out;
1993
1994 ext4_lock_group(sb, group);
Curt Wohlgemuth8a57d9d2010-05-16 15:00:00 -04001995
1996 /*
1997 * We need to check again after locking the
1998 * block group
1999 */
Alex Tomasc9de5602008-01-29 00:19:52 -05002000 if (!ext4_mb_good_group(ac, group, cr)) {
Alex Tomasc9de5602008-01-29 00:19:52 -05002001 ext4_unlock_group(sb, group);
Jing Zhange39e07f2010-05-14 00:00:00 -04002002 ext4_mb_unload_buddy(&e4b);
Alex Tomasc9de5602008-01-29 00:19:52 -05002003 continue;
2004 }
2005
2006 ac->ac_groups_scanned++;
Theodore Ts'o75507ef2009-05-01 12:58:36 -04002007 if (cr == 0)
Alex Tomasc9de5602008-01-29 00:19:52 -05002008 ext4_mb_simple_scan_group(ac, &e4b);
Eric Sandeen506bf2d2010-07-27 11:56:06 -04002009 else if (cr == 1 && sbi->s_stripe &&
2010 !(ac->ac_g_ex.fe_len % sbi->s_stripe))
Alex Tomasc9de5602008-01-29 00:19:52 -05002011 ext4_mb_scan_aligned(ac, &e4b);
2012 else
2013 ext4_mb_complex_scan_group(ac, &e4b);
2014
2015 ext4_unlock_group(sb, group);
Jing Zhange39e07f2010-05-14 00:00:00 -04002016 ext4_mb_unload_buddy(&e4b);
Alex Tomasc9de5602008-01-29 00:19:52 -05002017
2018 if (ac->ac_status != AC_STATUS_CONTINUE)
2019 break;
2020 }
2021 }
2022
2023 if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND &&
2024 !(ac->ac_flags & EXT4_MB_HINT_FIRST)) {
2025 /*
2026 * We've been searching too long. Let's try to allocate
2027 * the best chunk we've found so far
2028 */
2029
2030 ext4_mb_try_best_found(ac, &e4b);
2031 if (ac->ac_status != AC_STATUS_FOUND) {
2032 /*
2033 * Someone more lucky has already allocated it.
2034 * The only thing we can do is just take first
2035 * found block(s)
2036 printk(KERN_DEBUG "EXT4-fs: someone won our chunk\n");
2037 */
2038 ac->ac_b_ex.fe_group = 0;
2039 ac->ac_b_ex.fe_start = 0;
2040 ac->ac_b_ex.fe_len = 0;
2041 ac->ac_status = AC_STATUS_CONTINUE;
2042 ac->ac_flags |= EXT4_MB_HINT_FIRST;
2043 cr = 3;
2044 atomic_inc(&sbi->s_mb_lost_chunks);
2045 goto repeat;
2046 }
2047 }
2048out:
2049 return err;
2050}
2051
Alex Tomasc9de5602008-01-29 00:19:52 -05002052static void *ext4_mb_seq_groups_start(struct seq_file *seq, loff_t *pos)
2053{
2054 struct super_block *sb = seq->private;
Alex Tomasc9de5602008-01-29 00:19:52 -05002055 ext4_group_t group;
2056
Theodore Ts'o8df96752009-05-01 08:50:38 -04002057 if (*pos < 0 || *pos >= ext4_get_groups_count(sb))
Alex Tomasc9de5602008-01-29 00:19:52 -05002058 return NULL;
Alex Tomasc9de5602008-01-29 00:19:52 -05002059 group = *pos + 1;
Theodore Ts'oa9df9a42009-01-05 22:18:16 -05002060 return (void *) ((unsigned long) group);
Alex Tomasc9de5602008-01-29 00:19:52 -05002061}
2062
2063static void *ext4_mb_seq_groups_next(struct seq_file *seq, void *v, loff_t *pos)
2064{
2065 struct super_block *sb = seq->private;
Alex Tomasc9de5602008-01-29 00:19:52 -05002066 ext4_group_t group;
2067
2068 ++*pos;
Theodore Ts'o8df96752009-05-01 08:50:38 -04002069 if (*pos < 0 || *pos >= ext4_get_groups_count(sb))
Alex Tomasc9de5602008-01-29 00:19:52 -05002070 return NULL;
2071 group = *pos + 1;
Theodore Ts'oa9df9a42009-01-05 22:18:16 -05002072 return (void *) ((unsigned long) group);
Alex Tomasc9de5602008-01-29 00:19:52 -05002073}
2074
2075static int ext4_mb_seq_groups_show(struct seq_file *seq, void *v)
2076{
2077 struct super_block *sb = seq->private;
Theodore Ts'oa9df9a42009-01-05 22:18:16 -05002078 ext4_group_t group = (ext4_group_t) ((unsigned long) v);
Alex Tomasc9de5602008-01-29 00:19:52 -05002079 int i;
2080 int err;
2081 struct ext4_buddy e4b;
2082 struct sg {
2083 struct ext4_group_info info;
Eric Sandeena36b4492009-08-25 22:36:45 -04002084 ext4_grpblk_t counters[16];
Alex Tomasc9de5602008-01-29 00:19:52 -05002085 } sg;
2086
2087 group--;
2088 if (group == 0)
2089 seq_printf(seq, "#%-5s: %-5s %-5s %-5s "
2090 "[ %-5s %-5s %-5s %-5s %-5s %-5s %-5s "
2091 "%-5s %-5s %-5s %-5s %-5s %-5s %-5s ]\n",
2092 "group", "free", "frags", "first",
2093 "2^0", "2^1", "2^2", "2^3", "2^4", "2^5", "2^6",
2094 "2^7", "2^8", "2^9", "2^10", "2^11", "2^12", "2^13");
2095
2096 i = (sb->s_blocksize_bits + 2) * sizeof(sg.info.bb_counters[0]) +
2097 sizeof(struct ext4_group_info);
2098 err = ext4_mb_load_buddy(sb, group, &e4b);
2099 if (err) {
Theodore Ts'oa9df9a42009-01-05 22:18:16 -05002100 seq_printf(seq, "#%-5u: I/O error\n", group);
Alex Tomasc9de5602008-01-29 00:19:52 -05002101 return 0;
2102 }
2103 ext4_lock_group(sb, group);
2104 memcpy(&sg, ext4_get_group_info(sb, group), i);
2105 ext4_unlock_group(sb, group);
Jing Zhange39e07f2010-05-14 00:00:00 -04002106 ext4_mb_unload_buddy(&e4b);
Alex Tomasc9de5602008-01-29 00:19:52 -05002107
Theodore Ts'oa9df9a42009-01-05 22:18:16 -05002108 seq_printf(seq, "#%-5u: %-5u %-5u %-5u [", group, sg.info.bb_free,
Alex Tomasc9de5602008-01-29 00:19:52 -05002109 sg.info.bb_fragments, sg.info.bb_first_free);
2110 for (i = 0; i <= 13; i++)
2111 seq_printf(seq, " %-5u", i <= sb->s_blocksize_bits + 1 ?
2112 sg.info.bb_counters[i] : 0);
2113 seq_printf(seq, " ]\n");
2114
2115 return 0;
2116}
2117
2118static void ext4_mb_seq_groups_stop(struct seq_file *seq, void *v)
2119{
2120}
2121
Tobias Klauser7f1346a2009-09-05 09:28:54 -04002122static const struct seq_operations ext4_mb_seq_groups_ops = {
Alex Tomasc9de5602008-01-29 00:19:52 -05002123 .start = ext4_mb_seq_groups_start,
2124 .next = ext4_mb_seq_groups_next,
2125 .stop = ext4_mb_seq_groups_stop,
2126 .show = ext4_mb_seq_groups_show,
2127};
2128
2129static int ext4_mb_seq_groups_open(struct inode *inode, struct file *file)
2130{
2131 struct super_block *sb = PDE(inode)->data;
2132 int rc;
2133
2134 rc = seq_open(file, &ext4_mb_seq_groups_ops);
2135 if (rc == 0) {
Joe Perchesa271fe82010-07-27 11:56:04 -04002136 struct seq_file *m = file->private_data;
Alex Tomasc9de5602008-01-29 00:19:52 -05002137 m->private = sb;
2138 }
2139 return rc;
2140
2141}
2142
Tobias Klauser7f1346a2009-09-05 09:28:54 -04002143static const struct file_operations ext4_mb_seq_groups_fops = {
Alex Tomasc9de5602008-01-29 00:19:52 -05002144 .owner = THIS_MODULE,
2145 .open = ext4_mb_seq_groups_open,
2146 .read = seq_read,
2147 .llseek = seq_lseek,
2148 .release = seq_release,
2149};
2150
Curt Wohlgemuthfb1813f2010-10-27 21:29:12 -04002151static struct kmem_cache *get_groupinfo_cache(int blocksize_bits)
2152{
2153 int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE;
2154 struct kmem_cache *cachep = ext4_groupinfo_caches[cache_index];
2155
2156 BUG_ON(!cachep);
2157 return cachep;
2158}
Frederic Bohe5f21b0e2008-07-11 19:27:31 -04002159
2160/* Create and initialize ext4_group_info data for the given group. */
Aneesh Kumar K.V920313a2009-01-05 21:36:19 -05002161int ext4_mb_add_groupinfo(struct super_block *sb, ext4_group_t group,
Frederic Bohe5f21b0e2008-07-11 19:27:31 -04002162 struct ext4_group_desc *desc)
2163{
Curt Wohlgemuthfb1813f2010-10-27 21:29:12 -04002164 int i;
Frederic Bohe5f21b0e2008-07-11 19:27:31 -04002165 int metalen = 0;
2166 struct ext4_sb_info *sbi = EXT4_SB(sb);
2167 struct ext4_group_info **meta_group_info;
Curt Wohlgemuthfb1813f2010-10-27 21:29:12 -04002168 struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits);
Frederic Bohe5f21b0e2008-07-11 19:27:31 -04002169
2170 /*
2171 * First check if this group is the first of a reserved block.
2172 * If it's true, we have to allocate a new table of pointers
2173 * to ext4_group_info structures
2174 */
2175 if (group % EXT4_DESC_PER_BLOCK(sb) == 0) {
2176 metalen = sizeof(*meta_group_info) <<
2177 EXT4_DESC_PER_BLOCK_BITS(sb);
2178 meta_group_info = kmalloc(metalen, GFP_KERNEL);
2179 if (meta_group_info == NULL) {
Joe Perches7f6a11e2012-03-19 23:09:43 -04002180 ext4_msg(sb, KERN_ERR, "can't allocate mem "
Theodore Ts'o9d8b9ec2011-08-01 17:41:35 -04002181 "for a buddy group");
Frederic Bohe5f21b0e2008-07-11 19:27:31 -04002182 goto exit_meta_group_info;
2183 }
2184 sbi->s_group_info[group >> EXT4_DESC_PER_BLOCK_BITS(sb)] =
2185 meta_group_info;
2186 }
2187
Frederic Bohe5f21b0e2008-07-11 19:27:31 -04002188 meta_group_info =
2189 sbi->s_group_info[group >> EXT4_DESC_PER_BLOCK_BITS(sb)];
2190 i = group & (EXT4_DESC_PER_BLOCK(sb) - 1);
2191
Curt Wohlgemuthfb1813f2010-10-27 21:29:12 -04002192 meta_group_info[i] = kmem_cache_alloc(cachep, GFP_KERNEL);
Frederic Bohe5f21b0e2008-07-11 19:27:31 -04002193 if (meta_group_info[i] == NULL) {
Joe Perches7f6a11e2012-03-19 23:09:43 -04002194 ext4_msg(sb, KERN_ERR, "can't allocate buddy mem");
Frederic Bohe5f21b0e2008-07-11 19:27:31 -04002195 goto exit_group_info;
2196 }
Curt Wohlgemuthfb1813f2010-10-27 21:29:12 -04002197 memset(meta_group_info[i], 0, kmem_cache_size(cachep));
Frederic Bohe5f21b0e2008-07-11 19:27:31 -04002198 set_bit(EXT4_GROUP_INFO_NEED_INIT_BIT,
2199 &(meta_group_info[i]->bb_state));
2200
2201 /*
2202 * initialize bb_free to be able to skip
2203 * empty groups without initialization
2204 */
2205 if (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) {
2206 meta_group_info[i]->bb_free =
Theodore Ts'ocff1dfd72011-09-09 19:12:51 -04002207 ext4_free_clusters_after_init(sb, group, desc);
Frederic Bohe5f21b0e2008-07-11 19:27:31 -04002208 } else {
2209 meta_group_info[i]->bb_free =
Theodore Ts'o021b65b2011-09-09 19:08:51 -04002210 ext4_free_group_clusters(sb, desc);
Frederic Bohe5f21b0e2008-07-11 19:27:31 -04002211 }
2212
2213 INIT_LIST_HEAD(&meta_group_info[i]->bb_prealloc_list);
Aneesh Kumar K.V920313a2009-01-05 21:36:19 -05002214 init_rwsem(&meta_group_info[i]->alloc_sem);
Venkatesh Pallipadi64e290e2010-03-04 22:25:21 -05002215 meta_group_info[i]->bb_free_root = RB_ROOT;
Curt Wohlgemuth8a57d9d2010-05-16 15:00:00 -04002216 meta_group_info[i]->bb_largest_free_order = -1; /* uninit */
Frederic Bohe5f21b0e2008-07-11 19:27:31 -04002217
2218#ifdef DOUBLE_CHECK
2219 {
2220 struct buffer_head *bh;
2221 meta_group_info[i]->bb_bitmap =
2222 kmalloc(sb->s_blocksize, GFP_KERNEL);
2223 BUG_ON(meta_group_info[i]->bb_bitmap == NULL);
2224 bh = ext4_read_block_bitmap(sb, group);
2225 BUG_ON(bh == NULL);
2226 memcpy(meta_group_info[i]->bb_bitmap, bh->b_data,
2227 sb->s_blocksize);
2228 put_bh(bh);
2229 }
2230#endif
2231
2232 return 0;
2233
2234exit_group_info:
2235 /* If a meta_group_info table has been allocated, release it now */
Tao Macaaf7a22011-07-11 18:42:42 -04002236 if (group % EXT4_DESC_PER_BLOCK(sb) == 0) {
Frederic Bohe5f21b0e2008-07-11 19:27:31 -04002237 kfree(sbi->s_group_info[group >> EXT4_DESC_PER_BLOCK_BITS(sb)]);
Tao Macaaf7a22011-07-11 18:42:42 -04002238 sbi->s_group_info[group >> EXT4_DESC_PER_BLOCK_BITS(sb)] = NULL;
2239 }
Frederic Bohe5f21b0e2008-07-11 19:27:31 -04002240exit_meta_group_info:
2241 return -ENOMEM;
2242} /* ext4_mb_add_groupinfo */
2243
Alex Tomasc9de5602008-01-29 00:19:52 -05002244static int ext4_mb_init_backend(struct super_block *sb)
2245{
Theodore Ts'o8df96752009-05-01 08:50:38 -04002246 ext4_group_t ngroups = ext4_get_groups_count(sb);
Alex Tomasc9de5602008-01-29 00:19:52 -05002247 ext4_group_t i;
Alex Tomasc9de5602008-01-29 00:19:52 -05002248 struct ext4_sb_info *sbi = EXT4_SB(sb);
Frederic Bohe5f21b0e2008-07-11 19:27:31 -04002249 struct ext4_super_block *es = sbi->s_es;
2250 int num_meta_group_infos;
2251 int num_meta_group_infos_max;
2252 int array_size;
Frederic Bohe5f21b0e2008-07-11 19:27:31 -04002253 struct ext4_group_desc *desc;
Curt Wohlgemuthfb1813f2010-10-27 21:29:12 -04002254 struct kmem_cache *cachep;
Alex Tomasc9de5602008-01-29 00:19:52 -05002255
Frederic Bohe5f21b0e2008-07-11 19:27:31 -04002256 /* This is the number of blocks used by GDT */
Theodore Ts'o8df96752009-05-01 08:50:38 -04002257 num_meta_group_infos = (ngroups + EXT4_DESC_PER_BLOCK(sb) -
Frederic Bohe5f21b0e2008-07-11 19:27:31 -04002258 1) >> EXT4_DESC_PER_BLOCK_BITS(sb);
2259
2260 /*
2261 * This is the total number of blocks used by GDT including
2262 * the number of reserved blocks for GDT.
2263 * The s_group_info array is allocated with this value
2264 * to allow a clean online resize without a complex
2265 * manipulation of pointer.
2266 * The drawback is the unused memory when no resize
2267 * occurs but it's very low in terms of pages
2268 * (see comments below)
2269 * Need to handle this properly when META_BG resizing is allowed
2270 */
2271 num_meta_group_infos_max = num_meta_group_infos +
2272 le16_to_cpu(es->s_reserved_gdt_blocks);
2273
2274 /*
2275 * array_size is the size of s_group_info array. We round it
2276 * to the next power of two because this approximation is done
2277 * internally by kmalloc so we can have some more memory
2278 * for free here (e.g. may be used for META_BG resize).
2279 */
2280 array_size = 1;
2281 while (array_size < sizeof(*sbi->s_group_info) *
2282 num_meta_group_infos_max)
2283 array_size = array_size << 1;
Alex Tomasc9de5602008-01-29 00:19:52 -05002284 /* An 8TB filesystem with 64-bit pointers requires a 4096 byte
2285 * kmalloc. A 128kb malloc should suffice for a 256TB filesystem.
2286 * So a two level scheme suffices for now. */
Theodore Ts'of18a5f22011-08-01 08:45:38 -04002287 sbi->s_group_info = ext4_kvzalloc(array_size, GFP_KERNEL);
Alex Tomasc9de5602008-01-29 00:19:52 -05002288 if (sbi->s_group_info == NULL) {
Theodore Ts'o9d8b9ec2011-08-01 17:41:35 -04002289 ext4_msg(sb, KERN_ERR, "can't allocate buddy meta group");
Alex Tomasc9de5602008-01-29 00:19:52 -05002290 return -ENOMEM;
2291 }
2292 sbi->s_buddy_cache = new_inode(sb);
2293 if (sbi->s_buddy_cache == NULL) {
Theodore Ts'o9d8b9ec2011-08-01 17:41:35 -04002294 ext4_msg(sb, KERN_ERR, "can't get new inode");
Alex Tomasc9de5602008-01-29 00:19:52 -05002295 goto err_freesgi;
2296 }
Yu Jian48e60612011-08-01 17:41:39 -04002297 /* To avoid potentially colliding with an valid on-disk inode number,
2298 * use EXT4_BAD_INO for the buddy cache inode number. This inode is
2299 * not in the inode hash, so it should never be found by iget(), but
2300 * this will avoid confusion if it ever shows up during debugging. */
2301 sbi->s_buddy_cache->i_ino = EXT4_BAD_INO;
Alex Tomasc9de5602008-01-29 00:19:52 -05002302 EXT4_I(sbi->s_buddy_cache)->i_disksize = 0;
Theodore Ts'o8df96752009-05-01 08:50:38 -04002303 for (i = 0; i < ngroups; i++) {
Alex Tomasc9de5602008-01-29 00:19:52 -05002304 desc = ext4_get_group_desc(sb, i, NULL);
2305 if (desc == NULL) {
Theodore Ts'o9d8b9ec2011-08-01 17:41:35 -04002306 ext4_msg(sb, KERN_ERR, "can't read descriptor %u", i);
Alex Tomasc9de5602008-01-29 00:19:52 -05002307 goto err_freebuddy;
2308 }
Frederic Bohe5f21b0e2008-07-11 19:27:31 -04002309 if (ext4_mb_add_groupinfo(sb, i, desc) != 0)
2310 goto err_freebuddy;
Alex Tomasc9de5602008-01-29 00:19:52 -05002311 }
2312
2313 return 0;
2314
2315err_freebuddy:
Curt Wohlgemuthfb1813f2010-10-27 21:29:12 -04002316 cachep = get_groupinfo_cache(sb->s_blocksize_bits);
Roel Kluinf1fa3342008-04-29 22:01:15 -04002317 while (i-- > 0)
Curt Wohlgemuthfb1813f2010-10-27 21:29:12 -04002318 kmem_cache_free(cachep, ext4_get_group_info(sb, i));
Alex Tomasc9de5602008-01-29 00:19:52 -05002319 i = num_meta_group_infos;
Roel Kluinf1fa3342008-04-29 22:01:15 -04002320 while (i-- > 0)
Alex Tomasc9de5602008-01-29 00:19:52 -05002321 kfree(sbi->s_group_info[i]);
2322 iput(sbi->s_buddy_cache);
2323err_freesgi:
Theodore Ts'of18a5f22011-08-01 08:45:38 -04002324 ext4_kvfree(sbi->s_group_info);
Alex Tomasc9de5602008-01-29 00:19:52 -05002325 return -ENOMEM;
2326}
2327
Eric Sandeen2892c152011-02-12 08:12:18 -05002328static void ext4_groupinfo_destroy_slabs(void)
2329{
2330 int i;
2331
2332 for (i = 0; i < NR_GRPINFO_CACHES; i++) {
2333 if (ext4_groupinfo_caches[i])
2334 kmem_cache_destroy(ext4_groupinfo_caches[i]);
2335 ext4_groupinfo_caches[i] = NULL;
2336 }
2337}
2338
2339static int ext4_groupinfo_create_slab(size_t size)
2340{
2341 static DEFINE_MUTEX(ext4_grpinfo_slab_create_mutex);
2342 int slab_size;
2343 int blocksize_bits = order_base_2(size);
2344 int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE;
2345 struct kmem_cache *cachep;
2346
2347 if (cache_index >= NR_GRPINFO_CACHES)
2348 return -EINVAL;
2349
2350 if (unlikely(cache_index < 0))
2351 cache_index = 0;
2352
2353 mutex_lock(&ext4_grpinfo_slab_create_mutex);
2354 if (ext4_groupinfo_caches[cache_index]) {
2355 mutex_unlock(&ext4_grpinfo_slab_create_mutex);
2356 return 0; /* Already created */
2357 }
2358
2359 slab_size = offsetof(struct ext4_group_info,
2360 bb_counters[blocksize_bits + 2]);
2361
2362 cachep = kmem_cache_create(ext4_groupinfo_slab_names[cache_index],
2363 slab_size, 0, SLAB_RECLAIM_ACCOUNT,
2364 NULL);
2365
Tao Ma823ba012011-07-11 18:26:01 -04002366 ext4_groupinfo_caches[cache_index] = cachep;
2367
Eric Sandeen2892c152011-02-12 08:12:18 -05002368 mutex_unlock(&ext4_grpinfo_slab_create_mutex);
2369 if (!cachep) {
Theodore Ts'o9d8b9ec2011-08-01 17:41:35 -04002370 printk(KERN_EMERG
2371 "EXT4-fs: no memory for groupinfo slab cache\n");
Eric Sandeen2892c152011-02-12 08:12:18 -05002372 return -ENOMEM;
2373 }
2374
Eric Sandeen2892c152011-02-12 08:12:18 -05002375 return 0;
2376}
2377
Alex Tomasc9de5602008-01-29 00:19:52 -05002378int ext4_mb_init(struct super_block *sb, int needs_recovery)
2379{
2380 struct ext4_sb_info *sbi = EXT4_SB(sb);
Aneesh Kumar K.V6be2ded2008-07-23 14:14:05 -04002381 unsigned i, j;
Alex Tomasc9de5602008-01-29 00:19:52 -05002382 unsigned offset;
2383 unsigned max;
Shen Feng74767c52008-07-11 19:27:31 -04002384 int ret;
Alex Tomasc9de5602008-01-29 00:19:52 -05002385
Eric Sandeen19278052009-08-25 22:36:25 -04002386 i = (sb->s_blocksize_bits + 2) * sizeof(*sbi->s_mb_offsets);
Alex Tomasc9de5602008-01-29 00:19:52 -05002387
2388 sbi->s_mb_offsets = kmalloc(i, GFP_KERNEL);
2389 if (sbi->s_mb_offsets == NULL) {
Curt Wohlgemuthfb1813f2010-10-27 21:29:12 -04002390 ret = -ENOMEM;
2391 goto out;
Alex Tomasc9de5602008-01-29 00:19:52 -05002392 }
Yasunori Gotoff7ef322008-12-17 00:48:39 -05002393
Eric Sandeen19278052009-08-25 22:36:25 -04002394 i = (sb->s_blocksize_bits + 2) * sizeof(*sbi->s_mb_maxs);
Alex Tomasc9de5602008-01-29 00:19:52 -05002395 sbi->s_mb_maxs = kmalloc(i, GFP_KERNEL);
2396 if (sbi->s_mb_maxs == NULL) {
Curt Wohlgemuthfb1813f2010-10-27 21:29:12 -04002397 ret = -ENOMEM;
2398 goto out;
2399 }
2400
Eric Sandeen2892c152011-02-12 08:12:18 -05002401 ret = ext4_groupinfo_create_slab(sb->s_blocksize);
2402 if (ret < 0)
2403 goto out;
Alex Tomasc9de5602008-01-29 00:19:52 -05002404
2405 /* order 0 is regular bitmap */
2406 sbi->s_mb_maxs[0] = sb->s_blocksize << 3;
2407 sbi->s_mb_offsets[0] = 0;
2408
2409 i = 1;
2410 offset = 0;
2411 max = sb->s_blocksize << 2;
2412 do {
2413 sbi->s_mb_offsets[i] = offset;
2414 sbi->s_mb_maxs[i] = max;
2415 offset += 1 << (sb->s_blocksize_bits - i);
2416 max = max >> 1;
2417 i++;
2418 } while (i <= sb->s_blocksize_bits + 1);
2419
Alex Tomasc9de5602008-01-29 00:19:52 -05002420 spin_lock_init(&sbi->s_md_lock);
Alex Tomasc9de5602008-01-29 00:19:52 -05002421 spin_lock_init(&sbi->s_bal_lock);
2422
2423 sbi->s_mb_max_to_scan = MB_DEFAULT_MAX_TO_SCAN;
2424 sbi->s_mb_min_to_scan = MB_DEFAULT_MIN_TO_SCAN;
2425 sbi->s_mb_stats = MB_DEFAULT_STATS;
2426 sbi->s_mb_stream_request = MB_DEFAULT_STREAM_THRESHOLD;
2427 sbi->s_mb_order2_reqs = MB_DEFAULT_ORDER2_REQS;
Theodore Ts'o27baebb2011-09-09 19:02:51 -04002428 /*
2429 * The default group preallocation is 512, which for 4k block
2430 * sizes translates to 2 megabytes. However for bigalloc file
2431 * systems, this is probably too big (i.e, if the cluster size
2432 * is 1 megabyte, then group preallocation size becomes half a
2433 * gigabyte!). As a default, we will keep a two megabyte
2434 * group pralloc size for cluster sizes up to 64k, and after
2435 * that, we will force a minimum group preallocation size of
2436 * 32 clusters. This translates to 8 megs when the cluster
2437 * size is 256k, and 32 megs when the cluster size is 1 meg,
2438 * which seems reasonable as a default.
2439 */
2440 sbi->s_mb_group_prealloc = max(MB_DEFAULT_GROUP_PREALLOC >>
2441 sbi->s_cluster_bits, 32);
Dan Ehrenbergd7a1fee2011-07-17 21:11:30 -04002442 /*
2443 * If there is a s_stripe > 1, then we set the s_mb_group_prealloc
2444 * to the lowest multiple of s_stripe which is bigger than
2445 * the s_mb_group_prealloc as determined above. We want
2446 * the preallocation size to be an exact multiple of the
2447 * RAID stripe size so that preallocations don't fragment
2448 * the stripes.
2449 */
2450 if (sbi->s_stripe > 1) {
2451 sbi->s_mb_group_prealloc = roundup(
2452 sbi->s_mb_group_prealloc, sbi->s_stripe);
2453 }
Alex Tomasc9de5602008-01-29 00:19:52 -05002454
Eric Sandeen730c2132008-09-13 15:23:29 -04002455 sbi->s_locality_groups = alloc_percpu(struct ext4_locality_group);
Alex Tomasc9de5602008-01-29 00:19:52 -05002456 if (sbi->s_locality_groups == NULL) {
Curt Wohlgemuthfb1813f2010-10-27 21:29:12 -04002457 ret = -ENOMEM;
Tao Ma7aa0bae2011-10-06 10:22:28 -04002458 goto out_free_groupinfo_slab;
Alex Tomasc9de5602008-01-29 00:19:52 -05002459 }
Eric Sandeen730c2132008-09-13 15:23:29 -04002460 for_each_possible_cpu(i) {
Alex Tomasc9de5602008-01-29 00:19:52 -05002461 struct ext4_locality_group *lg;
Eric Sandeen730c2132008-09-13 15:23:29 -04002462 lg = per_cpu_ptr(sbi->s_locality_groups, i);
Alex Tomasc9de5602008-01-29 00:19:52 -05002463 mutex_init(&lg->lg_mutex);
Aneesh Kumar K.V6be2ded2008-07-23 14:14:05 -04002464 for (j = 0; j < PREALLOC_TB_SIZE; j++)
2465 INIT_LIST_HEAD(&lg->lg_prealloc_list[j]);
Alex Tomasc9de5602008-01-29 00:19:52 -05002466 spin_lock_init(&lg->lg_prealloc_lock);
2467 }
2468
Yu Jian79a77c52011-08-01 17:41:46 -04002469 /* init file for buddy data */
2470 ret = ext4_mb_init_backend(sb);
Tao Ma7aa0bae2011-10-06 10:22:28 -04002471 if (ret != 0)
2472 goto out_free_locality_groups;
Yu Jian79a77c52011-08-01 17:41:46 -04002473
Theodore Ts'o296c3552009-09-30 00:32:42 -04002474 if (sbi->s_proc)
2475 proc_create_data("mb_groups", S_IRUGO, sbi->s_proc,
2476 &ext4_mb_seq_groups_fops, sb);
Alex Tomasc9de5602008-01-29 00:19:52 -05002477
Tao Ma7aa0bae2011-10-06 10:22:28 -04002478 return 0;
2479
2480out_free_locality_groups:
2481 free_percpu(sbi->s_locality_groups);
2482 sbi->s_locality_groups = NULL;
2483out_free_groupinfo_slab:
2484 ext4_groupinfo_destroy_slabs();
Curt Wohlgemuthfb1813f2010-10-27 21:29:12 -04002485out:
Tao Ma7aa0bae2011-10-06 10:22:28 -04002486 kfree(sbi->s_mb_offsets);
2487 sbi->s_mb_offsets = NULL;
2488 kfree(sbi->s_mb_maxs);
2489 sbi->s_mb_maxs = NULL;
Curt Wohlgemuthfb1813f2010-10-27 21:29:12 -04002490 return ret;
Alex Tomasc9de5602008-01-29 00:19:52 -05002491}
2492
Aneesh Kumar K.V955ce5f2009-05-02 20:35:09 -04002493/* need to called with the ext4 group lock held */
Alex Tomasc9de5602008-01-29 00:19:52 -05002494static void ext4_mb_cleanup_pa(struct ext4_group_info *grp)
2495{
2496 struct ext4_prealloc_space *pa;
2497 struct list_head *cur, *tmp;
2498 int count = 0;
2499
2500 list_for_each_safe(cur, tmp, &grp->bb_prealloc_list) {
2501 pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list);
2502 list_del(&pa->pa_group_list);
2503 count++;
Aneesh Kumar K.V688f05a2008-10-13 12:14:14 -04002504 kmem_cache_free(ext4_pspace_cachep, pa);
Alex Tomasc9de5602008-01-29 00:19:52 -05002505 }
2506 if (count)
Theodore Ts'o6ba495e2009-09-18 13:38:55 -04002507 mb_debug(1, "mballoc: %u PAs left\n", count);
Alex Tomasc9de5602008-01-29 00:19:52 -05002508
2509}
2510
2511int ext4_mb_release(struct super_block *sb)
2512{
Theodore Ts'o8df96752009-05-01 08:50:38 -04002513 ext4_group_t ngroups = ext4_get_groups_count(sb);
Alex Tomasc9de5602008-01-29 00:19:52 -05002514 ext4_group_t i;
2515 int num_meta_group_infos;
2516 struct ext4_group_info *grinfo;
2517 struct ext4_sb_info *sbi = EXT4_SB(sb);
Curt Wohlgemuthfb1813f2010-10-27 21:29:12 -04002518 struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits);
Alex Tomasc9de5602008-01-29 00:19:52 -05002519
Alex Tomasc9de5602008-01-29 00:19:52 -05002520 if (sbi->s_group_info) {
Theodore Ts'o8df96752009-05-01 08:50:38 -04002521 for (i = 0; i < ngroups; i++) {
Alex Tomasc9de5602008-01-29 00:19:52 -05002522 grinfo = ext4_get_group_info(sb, i);
2523#ifdef DOUBLE_CHECK
2524 kfree(grinfo->bb_bitmap);
2525#endif
2526 ext4_lock_group(sb, i);
2527 ext4_mb_cleanup_pa(grinfo);
2528 ext4_unlock_group(sb, i);
Curt Wohlgemuthfb1813f2010-10-27 21:29:12 -04002529 kmem_cache_free(cachep, grinfo);
Alex Tomasc9de5602008-01-29 00:19:52 -05002530 }
Theodore Ts'o8df96752009-05-01 08:50:38 -04002531 num_meta_group_infos = (ngroups +
Alex Tomasc9de5602008-01-29 00:19:52 -05002532 EXT4_DESC_PER_BLOCK(sb) - 1) >>
2533 EXT4_DESC_PER_BLOCK_BITS(sb);
2534 for (i = 0; i < num_meta_group_infos; i++)
2535 kfree(sbi->s_group_info[i]);
Theodore Ts'of18a5f22011-08-01 08:45:38 -04002536 ext4_kvfree(sbi->s_group_info);
Alex Tomasc9de5602008-01-29 00:19:52 -05002537 }
2538 kfree(sbi->s_mb_offsets);
2539 kfree(sbi->s_mb_maxs);
2540 if (sbi->s_buddy_cache)
2541 iput(sbi->s_buddy_cache);
2542 if (sbi->s_mb_stats) {
Theodore Ts'o9d8b9ec2011-08-01 17:41:35 -04002543 ext4_msg(sb, KERN_INFO,
2544 "mballoc: %u blocks %u reqs (%u success)",
Alex Tomasc9de5602008-01-29 00:19:52 -05002545 atomic_read(&sbi->s_bal_allocated),
2546 atomic_read(&sbi->s_bal_reqs),
2547 atomic_read(&sbi->s_bal_success));
Theodore Ts'o9d8b9ec2011-08-01 17:41:35 -04002548 ext4_msg(sb, KERN_INFO,
2549 "mballoc: %u extents scanned, %u goal hits, "
2550 "%u 2^N hits, %u breaks, %u lost",
Alex Tomasc9de5602008-01-29 00:19:52 -05002551 atomic_read(&sbi->s_bal_ex_scanned),
2552 atomic_read(&sbi->s_bal_goals),
2553 atomic_read(&sbi->s_bal_2orders),
2554 atomic_read(&sbi->s_bal_breaks),
2555 atomic_read(&sbi->s_mb_lost_chunks));
Theodore Ts'o9d8b9ec2011-08-01 17:41:35 -04002556 ext4_msg(sb, KERN_INFO,
2557 "mballoc: %lu generated and it took %Lu",
Tao Maced156e2011-07-23 16:18:05 -04002558 sbi->s_mb_buddies_generated,
Alex Tomasc9de5602008-01-29 00:19:52 -05002559 sbi->s_mb_generation_time);
Theodore Ts'o9d8b9ec2011-08-01 17:41:35 -04002560 ext4_msg(sb, KERN_INFO,
2561 "mballoc: %u preallocated, %u discarded",
Alex Tomasc9de5602008-01-29 00:19:52 -05002562 atomic_read(&sbi->s_mb_preallocated),
2563 atomic_read(&sbi->s_mb_discarded));
2564 }
2565
Eric Sandeen730c2132008-09-13 15:23:29 -04002566 free_percpu(sbi->s_locality_groups);
Theodore Ts'o296c3552009-09-30 00:32:42 -04002567 if (sbi->s_proc)
2568 remove_proc_entry("mb_groups", sbi->s_proc);
Alex Tomasc9de5602008-01-29 00:19:52 -05002569
2570 return 0;
2571}
2572
Lukas Czerner77ca6cd2010-10-27 21:30:11 -04002573static inline int ext4_issue_discard(struct super_block *sb,
Theodore Ts'o84130192011-09-09 18:50:51 -04002574 ext4_group_t block_group, ext4_grpblk_t cluster, int count)
Jiaying Zhang5c521832010-07-27 11:56:05 -04002575{
Jiaying Zhang5c521832010-07-27 11:56:05 -04002576 ext4_fsblk_t discard_block;
2577
Theodore Ts'o84130192011-09-09 18:50:51 -04002578 discard_block = (EXT4_C2B(EXT4_SB(sb), cluster) +
2579 ext4_group_first_block_no(sb, block_group));
2580 count = EXT4_C2B(EXT4_SB(sb), count);
Jiaying Zhang5c521832010-07-27 11:56:05 -04002581 trace_ext4_discard_blocks(sb,
2582 (unsigned long long) discard_block, count);
Lukas Czerner93259632011-01-10 12:09:59 -05002583 return sb_issue_discard(sb, discard_block, count, GFP_NOFS, 0);
Jiaying Zhang5c521832010-07-27 11:56:05 -04002584}
2585
Theodore Ts'o3e624fc2008-10-16 20:00:24 -04002586/*
2587 * This function is called by the jbd2 layer once the commit has finished,
2588 * so we know we can free the blocks that were released with that commit.
2589 */
Bobi Jam18aadd42012-02-20 17:53:02 -05002590static void ext4_free_data_callback(struct super_block *sb,
2591 struct ext4_journal_cb_entry *jce,
2592 int rc)
Alex Tomasc9de5602008-01-29 00:19:52 -05002593{
Bobi Jam18aadd42012-02-20 17:53:02 -05002594 struct ext4_free_data *entry = (struct ext4_free_data *)jce;
Alex Tomasc9de5602008-01-29 00:19:52 -05002595 struct ext4_buddy e4b;
Aneesh Kumar K.Vc8940582008-10-16 10:14:27 -04002596 struct ext4_group_info *db;
Theodore Ts'od9f34502011-04-30 13:47:24 -04002597 int err, count = 0, count2 = 0;
Alex Tomasc9de5602008-01-29 00:19:52 -05002598
Bobi Jam18aadd42012-02-20 17:53:02 -05002599 mb_debug(1, "gonna free %u blocks in group %u (0x%p):",
2600 entry->efd_count, entry->efd_group, entry);
Alex Tomasc9de5602008-01-29 00:19:52 -05002601
Bobi Jam18aadd42012-02-20 17:53:02 -05002602 if (test_opt(sb, DISCARD))
2603 ext4_issue_discard(sb, entry->efd_group,
2604 entry->efd_start_cluster, entry->efd_count);
Alex Tomasc9de5602008-01-29 00:19:52 -05002605
Bobi Jam18aadd42012-02-20 17:53:02 -05002606 err = ext4_mb_load_buddy(sb, entry->efd_group, &e4b);
2607 /* we expect to find existing buddy because it's pinned */
2608 BUG_ON(err != 0);
Theodore Ts'ob90f6872010-04-20 16:51:59 -04002609
Alex Tomasc9de5602008-01-29 00:19:52 -05002610
Bobi Jam18aadd42012-02-20 17:53:02 -05002611 db = e4b.bd_info;
2612 /* there are blocks to put in buddy to make them really free */
2613 count += entry->efd_count;
2614 count2++;
2615 ext4_lock_group(sb, entry->efd_group);
2616 /* Take it out of per group rb tree */
2617 rb_erase(&entry->efd_node, &(db->bb_free_root));
2618 mb_free_blocks(NULL, &e4b, entry->efd_start_cluster, entry->efd_count);
Aneesh Kumar K.Vc8940582008-10-16 10:14:27 -04002619
Bobi Jam18aadd42012-02-20 17:53:02 -05002620 /*
2621 * Clear the trimmed flag for the group so that the next
2622 * ext4_trim_fs can trim it.
2623 * If the volume is mounted with -o discard, online discard
2624 * is supported and the free blocks will be trimmed online.
2625 */
2626 if (!test_opt(sb, DISCARD))
2627 EXT4_MB_GRP_CLEAR_TRIMMED(db);
2628
2629 if (!db->bb_free_root.rb_node) {
2630 /* No more items in the per group rb tree
2631 * balance refcounts from ext4_mb_free_metadata()
Tao Ma3d56b8d2011-07-11 00:03:38 -04002632 */
Bobi Jam18aadd42012-02-20 17:53:02 -05002633 page_cache_release(e4b.bd_buddy_page);
2634 page_cache_release(e4b.bd_bitmap_page);
Theodore Ts'o3e624fc2008-10-16 20:00:24 -04002635 }
Bobi Jam18aadd42012-02-20 17:53:02 -05002636 ext4_unlock_group(sb, entry->efd_group);
2637 kmem_cache_free(ext4_free_data_cachep, entry);
2638 ext4_mb_unload_buddy(&e4b);
Alex Tomasc9de5602008-01-29 00:19:52 -05002639
Theodore Ts'o6ba495e2009-09-18 13:38:55 -04002640 mb_debug(1, "freed %u blocks in %u structures\n", count, count2);
Alex Tomasc9de5602008-01-29 00:19:52 -05002641}
2642
Theodore Ts'o6ba495e2009-09-18 13:38:55 -04002643#ifdef CONFIG_EXT4_DEBUG
2644u8 mb_enable_debug __read_mostly;
2645
2646static struct dentry *debugfs_dir;
2647static struct dentry *debugfs_debug;
2648
2649static void __init ext4_create_debugfs_entry(void)
2650{
2651 debugfs_dir = debugfs_create_dir("ext4", NULL);
2652 if (debugfs_dir)
2653 debugfs_debug = debugfs_create_u8("mballoc-debug",
2654 S_IRUGO | S_IWUSR,
2655 debugfs_dir,
2656 &mb_enable_debug);
2657}
2658
2659static void ext4_remove_debugfs_entry(void)
2660{
2661 debugfs_remove(debugfs_debug);
2662 debugfs_remove(debugfs_dir);
2663}
2664
2665#else
2666
2667static void __init ext4_create_debugfs_entry(void)
2668{
2669}
2670
2671static void ext4_remove_debugfs_entry(void)
2672{
2673}
2674
2675#endif
2676
Theodore Ts'o5dabfc72010-10-27 21:30:14 -04002677int __init ext4_init_mballoc(void)
Alex Tomasc9de5602008-01-29 00:19:52 -05002678{
Theodore Ts'o16828082010-10-27 21:30:09 -04002679 ext4_pspace_cachep = KMEM_CACHE(ext4_prealloc_space,
2680 SLAB_RECLAIM_ACCOUNT);
Alex Tomasc9de5602008-01-29 00:19:52 -05002681 if (ext4_pspace_cachep == NULL)
2682 return -ENOMEM;
2683
Theodore Ts'o16828082010-10-27 21:30:09 -04002684 ext4_ac_cachep = KMEM_CACHE(ext4_allocation_context,
2685 SLAB_RECLAIM_ACCOUNT);
Eric Sandeen256bdb42008-02-10 01:13:33 -05002686 if (ext4_ac_cachep == NULL) {
2687 kmem_cache_destroy(ext4_pspace_cachep);
2688 return -ENOMEM;
2689 }
Aneesh Kumar K.Vc8940582008-10-16 10:14:27 -04002690
Bobi Jam18aadd42012-02-20 17:53:02 -05002691 ext4_free_data_cachep = KMEM_CACHE(ext4_free_data,
2692 SLAB_RECLAIM_ACCOUNT);
2693 if (ext4_free_data_cachep == NULL) {
Aneesh Kumar K.Vc8940582008-10-16 10:14:27 -04002694 kmem_cache_destroy(ext4_pspace_cachep);
2695 kmem_cache_destroy(ext4_ac_cachep);
2696 return -ENOMEM;
2697 }
Theodore Ts'o6ba495e2009-09-18 13:38:55 -04002698 ext4_create_debugfs_entry();
Alex Tomasc9de5602008-01-29 00:19:52 -05002699 return 0;
2700}
2701
Theodore Ts'o5dabfc72010-10-27 21:30:14 -04002702void ext4_exit_mballoc(void)
Alex Tomasc9de5602008-01-29 00:19:52 -05002703{
Theodore Ts'o60e66792010-05-17 07:00:00 -04002704 /*
Jesper Dangaard Brouer3e03f9c2009-07-05 22:29:27 -04002705 * Wait for completion of call_rcu()'s on ext4_pspace_cachep
2706 * before destroying the slab cache.
2707 */
2708 rcu_barrier();
Alex Tomasc9de5602008-01-29 00:19:52 -05002709 kmem_cache_destroy(ext4_pspace_cachep);
Eric Sandeen256bdb42008-02-10 01:13:33 -05002710 kmem_cache_destroy(ext4_ac_cachep);
Bobi Jam18aadd42012-02-20 17:53:02 -05002711 kmem_cache_destroy(ext4_free_data_cachep);
Eric Sandeen2892c152011-02-12 08:12:18 -05002712 ext4_groupinfo_destroy_slabs();
Theodore Ts'o6ba495e2009-09-18 13:38:55 -04002713 ext4_remove_debugfs_entry();
Alex Tomasc9de5602008-01-29 00:19:52 -05002714}
2715
2716
2717/*
Uwe Kleine-König73b2c712010-07-30 21:02:47 +02002718 * Check quota and mark chosen space (ac->ac_b_ex) non-free in bitmaps
Alex Tomasc9de5602008-01-29 00:19:52 -05002719 * Returns 0 if success or error code
2720 */
Eric Sandeen4ddfef72008-04-29 08:11:12 -04002721static noinline_for_stack int
2722ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac,
Theodore Ts'o53accfa2011-09-09 18:48:51 -04002723 handle_t *handle, unsigned int reserv_clstrs)
Alex Tomasc9de5602008-01-29 00:19:52 -05002724{
2725 struct buffer_head *bitmap_bh = NULL;
Alex Tomasc9de5602008-01-29 00:19:52 -05002726 struct ext4_group_desc *gdp;
2727 struct buffer_head *gdp_bh;
2728 struct ext4_sb_info *sbi;
2729 struct super_block *sb;
2730 ext4_fsblk_t block;
Aneesh Kumar K.V519deca2008-05-15 14:43:20 -04002731 int err, len;
Alex Tomasc9de5602008-01-29 00:19:52 -05002732
2733 BUG_ON(ac->ac_status != AC_STATUS_FOUND);
2734 BUG_ON(ac->ac_b_ex.fe_len <= 0);
2735
2736 sb = ac->ac_sb;
2737 sbi = EXT4_SB(sb);
Alex Tomasc9de5602008-01-29 00:19:52 -05002738
2739 err = -EIO;
Theodore Ts'o574ca172008-07-11 19:27:31 -04002740 bitmap_bh = ext4_read_block_bitmap(sb, ac->ac_b_ex.fe_group);
Alex Tomasc9de5602008-01-29 00:19:52 -05002741 if (!bitmap_bh)
2742 goto out_err;
2743
2744 err = ext4_journal_get_write_access(handle, bitmap_bh);
2745 if (err)
2746 goto out_err;
2747
2748 err = -EIO;
2749 gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh);
2750 if (!gdp)
2751 goto out_err;
2752
Theodore Ts'oa9df9a42009-01-05 22:18:16 -05002753 ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group,
Theodore Ts'o021b65b2011-09-09 19:08:51 -04002754 ext4_free_group_clusters(sb, gdp));
Aneesh Kumar K.V03cddb82008-06-05 20:59:29 -04002755
Alex Tomasc9de5602008-01-29 00:19:52 -05002756 err = ext4_journal_get_write_access(handle, gdp_bh);
2757 if (err)
2758 goto out_err;
2759
Akinobu Mitabda00de2010-03-03 23:53:25 -05002760 block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);
Alex Tomasc9de5602008-01-29 00:19:52 -05002761
Theodore Ts'o53accfa2011-09-09 18:48:51 -04002762 len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len);
Theodore Ts'o6fd058f2009-05-17 15:38:01 -04002763 if (!ext4_data_block_valid(sbi, block, len)) {
Eric Sandeen12062dd2010-02-15 14:19:27 -05002764 ext4_error(sb, "Allocating blocks %llu-%llu which overlap "
Theodore Ts'o1084f252012-03-19 23:13:43 -04002765 "fs metadata", block, block+len);
Aneesh Kumar K.V519deca2008-05-15 14:43:20 -04002766 /* File system mounted not to panic on error
2767 * Fix the bitmap and repeat the block allocation
2768 * We leak some of the blocks here.
2769 */
Aneesh Kumar K.V955ce5f2009-05-02 20:35:09 -04002770 ext4_lock_group(sb, ac->ac_b_ex.fe_group);
Yongqiang Yangc3e94d12011-07-26 22:05:53 -04002771 ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start,
2772 ac->ac_b_ex.fe_len);
Aneesh Kumar K.V955ce5f2009-05-02 20:35:09 -04002773 ext4_unlock_group(sb, ac->ac_b_ex.fe_group);
Frank Mayhar03901312009-01-07 00:06:22 -05002774 err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh);
Aneesh Kumar K.V519deca2008-05-15 14:43:20 -04002775 if (!err)
2776 err = -EAGAIN;
2777 goto out_err;
Alex Tomasc9de5602008-01-29 00:19:52 -05002778 }
Aneesh Kumar K.V955ce5f2009-05-02 20:35:09 -04002779
2780 ext4_lock_group(sb, ac->ac_b_ex.fe_group);
Alex Tomasc9de5602008-01-29 00:19:52 -05002781#ifdef AGGRESSIVE_CHECK
2782 {
2783 int i;
2784 for (i = 0; i < ac->ac_b_ex.fe_len; i++) {
2785 BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i,
2786 bitmap_bh->b_data));
2787 }
2788 }
2789#endif
Yongqiang Yangc3e94d12011-07-26 22:05:53 -04002790 ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start,
2791 ac->ac_b_ex.fe_len);
Alex Tomasc9de5602008-01-29 00:19:52 -05002792 if (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) {
2793 gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT);
Theodore Ts'o021b65b2011-09-09 19:08:51 -04002794 ext4_free_group_clusters_set(sb, gdp,
Theodore Ts'ocff1dfd72011-09-09 19:12:51 -04002795 ext4_free_clusters_after_init(sb,
Theodore Ts'o021b65b2011-09-09 19:08:51 -04002796 ac->ac_b_ex.fe_group, gdp));
Alex Tomasc9de5602008-01-29 00:19:52 -05002797 }
Theodore Ts'o021b65b2011-09-09 19:08:51 -04002798 len = ext4_free_group_clusters(sb, gdp) - ac->ac_b_ex.fe_len;
2799 ext4_free_group_clusters_set(sb, gdp, len);
Alex Tomasc9de5602008-01-29 00:19:52 -05002800 gdp->bg_checksum = ext4_group_desc_csum(sbi, ac->ac_b_ex.fe_group, gdp);
Aneesh Kumar K.V955ce5f2009-05-02 20:35:09 -04002801
2802 ext4_unlock_group(sb, ac->ac_b_ex.fe_group);
Theodore Ts'o57042652011-09-09 18:56:51 -04002803 percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len);
Mingming Caod2a17632008-07-14 17:52:37 -04002804 /*
Aneesh Kumar K.V6bc6e632008-10-10 09:39:00 -04002805 * Now reduce the dirty block count also. Should not go negative
Mingming Caod2a17632008-07-14 17:52:37 -04002806 */
Aneesh Kumar K.V6bc6e632008-10-10 09:39:00 -04002807 if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED))
2808 /* release all the reserved blocks if non delalloc */
Theodore Ts'o57042652011-09-09 18:56:51 -04002809 percpu_counter_sub(&sbi->s_dirtyclusters_counter,
2810 reserv_clstrs);
Alex Tomasc9de5602008-01-29 00:19:52 -05002811
Jose R. Santos772cb7c2008-07-11 19:27:31 -04002812 if (sbi->s_log_groups_per_flex) {
2813 ext4_group_t flex_group = ext4_flex_group(sbi,
2814 ac->ac_b_ex.fe_group);
Theodore Ts'o9f24e422009-03-04 19:09:10 -05002815 atomic_sub(ac->ac_b_ex.fe_len,
Theodore Ts'o24aaa8e2011-09-09 18:58:51 -04002816 &sbi->s_flex_groups[flex_group].free_clusters);
Jose R. Santos772cb7c2008-07-11 19:27:31 -04002817 }
2818
Frank Mayhar03901312009-01-07 00:06:22 -05002819 err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh);
Alex Tomasc9de5602008-01-29 00:19:52 -05002820 if (err)
2821 goto out_err;
Frank Mayhar03901312009-01-07 00:06:22 -05002822 err = ext4_handle_dirty_metadata(handle, NULL, gdp_bh);
Alex Tomasc9de5602008-01-29 00:19:52 -05002823
2824out_err:
Theodore Ts'oa0375152010-06-11 23:14:04 -04002825 ext4_mark_super_dirty(sb);
Aneesh Kumar K.V42a10ad2008-02-10 01:07:28 -05002826 brelse(bitmap_bh);
Alex Tomasc9de5602008-01-29 00:19:52 -05002827 return err;
2828}
2829
2830/*
2831 * here we normalize request for locality group
Dan Ehrenbergd7a1fee2011-07-17 21:11:30 -04002832 * Group request are normalized to s_mb_group_prealloc, which goes to
2833 * s_strip if we set the same via mount option.
2834 * s_mb_group_prealloc can be configured via
Theodore Ts'ob713a5e2009-03-31 09:11:14 -04002835 * /sys/fs/ext4/<partition>/mb_group_prealloc
Alex Tomasc9de5602008-01-29 00:19:52 -05002836 *
2837 * XXX: should we try to preallocate more than the group has now?
2838 */
2839static void ext4_mb_normalize_group_request(struct ext4_allocation_context *ac)
2840{
2841 struct super_block *sb = ac->ac_sb;
2842 struct ext4_locality_group *lg = ac->ac_lg;
2843
2844 BUG_ON(lg == NULL);
Dan Ehrenbergd7a1fee2011-07-17 21:11:30 -04002845 ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc;
Theodore Ts'o6ba495e2009-09-18 13:38:55 -04002846 mb_debug(1, "#%u: goal %u blocks for locality group\n",
Alex Tomasc9de5602008-01-29 00:19:52 -05002847 current->pid, ac->ac_g_ex.fe_len);
2848}
2849
2850/*
2851 * Normalization means making request better in terms of
2852 * size and alignment
2853 */
Eric Sandeen4ddfef72008-04-29 08:11:12 -04002854static noinline_for_stack void
2855ext4_mb_normalize_request(struct ext4_allocation_context *ac,
Alex Tomasc9de5602008-01-29 00:19:52 -05002856 struct ext4_allocation_request *ar)
2857{
Theodore Ts'o53accfa2011-09-09 18:48:51 -04002858 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
Alex Tomasc9de5602008-01-29 00:19:52 -05002859 int bsbits, max;
2860 ext4_lblk_t end;
Curt Wohlgemuth1592d2c2012-02-20 17:53:03 -05002861 loff_t size, start_off;
2862 loff_t orig_size __maybe_unused;
Andi Kleen5a0790c2010-06-14 13:28:03 -04002863 ext4_lblk_t start;
Alex Tomasc9de5602008-01-29 00:19:52 -05002864 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);
Aneesh Kumar K.V9a0762c2008-04-17 10:38:59 -04002865 struct ext4_prealloc_space *pa;
Alex Tomasc9de5602008-01-29 00:19:52 -05002866
2867 /* do normalize only data requests, metadata requests
2868 do not need preallocation */
2869 if (!(ac->ac_flags & EXT4_MB_HINT_DATA))
2870 return;
2871
2872 /* sometime caller may want exact blocks */
2873 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))
2874 return;
2875
2876 /* caller may indicate that preallocation isn't
2877 * required (it's a tail, for example) */
2878 if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC)
2879 return;
2880
2881 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) {
2882 ext4_mb_normalize_group_request(ac);
2883 return ;
2884 }
2885
2886 bsbits = ac->ac_sb->s_blocksize_bits;
2887
2888 /* first, let's learn actual file size
2889 * given current request is allocated */
Theodore Ts'o53accfa2011-09-09 18:48:51 -04002890 size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len);
Alex Tomasc9de5602008-01-29 00:19:52 -05002891 size = size << bsbits;
2892 if (size < i_size_read(ac->ac_inode))
2893 size = i_size_read(ac->ac_inode);
Andi Kleen5a0790c2010-06-14 13:28:03 -04002894 orig_size = size;
Alex Tomasc9de5602008-01-29 00:19:52 -05002895
Valerie Clement19304792008-05-13 19:31:14 -04002896 /* max size of free chunks */
2897 max = 2 << bsbits;
Alex Tomasc9de5602008-01-29 00:19:52 -05002898
Valerie Clement19304792008-05-13 19:31:14 -04002899#define NRL_CHECK_SIZE(req, size, max, chunk_size) \
2900 (req <= (size) || max <= (chunk_size))
Alex Tomasc9de5602008-01-29 00:19:52 -05002901
2902 /* first, try to predict filesize */
2903 /* XXX: should this table be tunable? */
2904 start_off = 0;
2905 if (size <= 16 * 1024) {
2906 size = 16 * 1024;
2907 } else if (size <= 32 * 1024) {
2908 size = 32 * 1024;
2909 } else if (size <= 64 * 1024) {
2910 size = 64 * 1024;
2911 } else if (size <= 128 * 1024) {
2912 size = 128 * 1024;
2913 } else if (size <= 256 * 1024) {
2914 size = 256 * 1024;
2915 } else if (size <= 512 * 1024) {
2916 size = 512 * 1024;
2917 } else if (size <= 1024 * 1024) {
2918 size = 1024 * 1024;
Valerie Clement19304792008-05-13 19:31:14 -04002919 } else if (NRL_CHECK_SIZE(size, 4 * 1024 * 1024, max, 2 * 1024)) {
Alex Tomasc9de5602008-01-29 00:19:52 -05002920 start_off = ((loff_t)ac->ac_o_ex.fe_logical >>
Valerie Clement19304792008-05-13 19:31:14 -04002921 (21 - bsbits)) << 21;
2922 size = 2 * 1024 * 1024;
2923 } else if (NRL_CHECK_SIZE(size, 8 * 1024 * 1024, max, 4 * 1024)) {
Alex Tomasc9de5602008-01-29 00:19:52 -05002924 start_off = ((loff_t)ac->ac_o_ex.fe_logical >>
2925 (22 - bsbits)) << 22;
2926 size = 4 * 1024 * 1024;
2927 } else if (NRL_CHECK_SIZE(ac->ac_o_ex.fe_len,
Valerie Clement19304792008-05-13 19:31:14 -04002928 (8<<20)>>bsbits, max, 8 * 1024)) {
Alex Tomasc9de5602008-01-29 00:19:52 -05002929 start_off = ((loff_t)ac->ac_o_ex.fe_logical >>
2930 (23 - bsbits)) << 23;
2931 size = 8 * 1024 * 1024;
2932 } else {
2933 start_off = (loff_t)ac->ac_o_ex.fe_logical << bsbits;
2934 size = ac->ac_o_ex.fe_len << bsbits;
2935 }
Andi Kleen5a0790c2010-06-14 13:28:03 -04002936 size = size >> bsbits;
2937 start = start_off >> bsbits;
Alex Tomasc9de5602008-01-29 00:19:52 -05002938
2939 /* don't cover already allocated blocks in selected range */
2940 if (ar->pleft && start <= ar->lleft) {
2941 size -= ar->lleft + 1 - start;
2942 start = ar->lleft + 1;
2943 }
2944 if (ar->pright && start + size - 1 >= ar->lright)
2945 size -= start + size - ar->lright;
2946
2947 end = start + size;
2948
2949 /* check we don't cross already preallocated blocks */
2950 rcu_read_lock();
Aneesh Kumar K.V9a0762c2008-04-17 10:38:59 -04002951 list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) {
Theodore Ts'o498e5f22008-11-05 00:14:04 -05002952 ext4_lblk_t pa_end;
Alex Tomasc9de5602008-01-29 00:19:52 -05002953
Alex Tomasc9de5602008-01-29 00:19:52 -05002954 if (pa->pa_deleted)
2955 continue;
2956 spin_lock(&pa->pa_lock);
2957 if (pa->pa_deleted) {
2958 spin_unlock(&pa->pa_lock);
2959 continue;
2960 }
2961
Theodore Ts'o53accfa2011-09-09 18:48:51 -04002962 pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb),
2963 pa->pa_len);
Alex Tomasc9de5602008-01-29 00:19:52 -05002964
2965 /* PA must not overlap original request */
2966 BUG_ON(!(ac->ac_o_ex.fe_logical >= pa_end ||
2967 ac->ac_o_ex.fe_logical < pa->pa_lstart));
2968
Eric Sandeen38877f42009-08-17 23:55:24 -04002969 /* skip PAs this normalized request doesn't overlap with */
2970 if (pa->pa_lstart >= end || pa_end <= start) {
Alex Tomasc9de5602008-01-29 00:19:52 -05002971 spin_unlock(&pa->pa_lock);
2972 continue;
2973 }
2974 BUG_ON(pa->pa_lstart <= start && pa_end >= end);
2975
Eric Sandeen38877f42009-08-17 23:55:24 -04002976 /* adjust start or end to be adjacent to this pa */
Alex Tomasc9de5602008-01-29 00:19:52 -05002977 if (pa_end <= ac->ac_o_ex.fe_logical) {
2978 BUG_ON(pa_end < start);
2979 start = pa_end;
Eric Sandeen38877f42009-08-17 23:55:24 -04002980 } else if (pa->pa_lstart > ac->ac_o_ex.fe_logical) {
Alex Tomasc9de5602008-01-29 00:19:52 -05002981 BUG_ON(pa->pa_lstart > end);
2982 end = pa->pa_lstart;
2983 }
2984 spin_unlock(&pa->pa_lock);
2985 }
2986 rcu_read_unlock();
2987 size = end - start;
2988
2989 /* XXX: extra loop to check we really don't overlap preallocations */
2990 rcu_read_lock();
Aneesh Kumar K.V9a0762c2008-04-17 10:38:59 -04002991 list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) {
Theodore Ts'o498e5f22008-11-05 00:14:04 -05002992 ext4_lblk_t pa_end;
Theodore Ts'o53accfa2011-09-09 18:48:51 -04002993
Alex Tomasc9de5602008-01-29 00:19:52 -05002994 spin_lock(&pa->pa_lock);
2995 if (pa->pa_deleted == 0) {
Theodore Ts'o53accfa2011-09-09 18:48:51 -04002996 pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb),
2997 pa->pa_len);
Alex Tomasc9de5602008-01-29 00:19:52 -05002998 BUG_ON(!(start >= pa_end || end <= pa->pa_lstart));
2999 }
3000 spin_unlock(&pa->pa_lock);
3001 }
3002 rcu_read_unlock();
3003
3004 if (start + size <= ac->ac_o_ex.fe_logical &&
3005 start > ac->ac_o_ex.fe_logical) {
Theodore Ts'o9d8b9ec2011-08-01 17:41:35 -04003006 ext4_msg(ac->ac_sb, KERN_ERR,
3007 "start %lu, size %lu, fe_logical %lu",
3008 (unsigned long) start, (unsigned long) size,
3009 (unsigned long) ac->ac_o_ex.fe_logical);
Alex Tomasc9de5602008-01-29 00:19:52 -05003010 }
3011 BUG_ON(start + size <= ac->ac_o_ex.fe_logical &&
3012 start > ac->ac_o_ex.fe_logical);
Theodore Ts'o7137d7a2011-09-09 18:38:51 -04003013 BUG_ON(size <= 0 || size > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb));
Alex Tomasc9de5602008-01-29 00:19:52 -05003014
3015 /* now prepare goal request */
3016
3017 /* XXX: is it better to align blocks WRT to logical
3018 * placement or satisfy big request as is */
3019 ac->ac_g_ex.fe_logical = start;
Theodore Ts'o53accfa2011-09-09 18:48:51 -04003020 ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size);
Alex Tomasc9de5602008-01-29 00:19:52 -05003021
3022 /* define goal start in order to merge */
3023 if (ar->pright && (ar->lright == (start + size))) {
3024 /* merge to the right */
3025 ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size,
3026 &ac->ac_f_ex.fe_group,
3027 &ac->ac_f_ex.fe_start);
3028 ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL;
3029 }
3030 if (ar->pleft && (ar->lleft + 1 == start)) {
3031 /* merge to the left */
3032 ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1,
3033 &ac->ac_f_ex.fe_group,
3034 &ac->ac_f_ex.fe_start);
3035 ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL;
3036 }
3037
Theodore Ts'o6ba495e2009-09-18 13:38:55 -04003038 mb_debug(1, "goal: %u(was %u) blocks at %u\n", (unsigned) size,
Alex Tomasc9de5602008-01-29 00:19:52 -05003039 (unsigned) orig_size, (unsigned) start);
3040}
3041
3042static void ext4_mb_collect_stats(struct ext4_allocation_context *ac)
3043{
3044 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
3045
3046 if (sbi->s_mb_stats && ac->ac_g_ex.fe_len > 1) {
3047 atomic_inc(&sbi->s_bal_reqs);
3048 atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated);
Curt Wohlgemuth291dae42010-05-16 16:00:00 -04003049 if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len)
Alex Tomasc9de5602008-01-29 00:19:52 -05003050 atomic_inc(&sbi->s_bal_success);
3051 atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned);
3052 if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start &&
3053 ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group)
3054 atomic_inc(&sbi->s_bal_goals);
3055 if (ac->ac_found > sbi->s_mb_max_to_scan)
3056 atomic_inc(&sbi->s_bal_breaks);
3057 }
3058
Theodore Ts'o296c3552009-09-30 00:32:42 -04003059 if (ac->ac_op == EXT4_MB_HISTORY_ALLOC)
3060 trace_ext4_mballoc_alloc(ac);
3061 else
3062 trace_ext4_mballoc_prealloc(ac);
Alex Tomasc9de5602008-01-29 00:19:52 -05003063}
3064
3065/*
Curt Wohlgemuthb8441672009-12-08 22:18:25 -05003066 * Called on failure; free up any blocks from the inode PA for this
3067 * context. We don't need this for MB_GROUP_PA because we only change
3068 * pa_free in ext4_mb_release_context(), but on failure, we've already
3069 * zeroed out ac->ac_b_ex.fe_len, so group_pa->pa_free is not changed.
3070 */
3071static void ext4_discard_allocated_blocks(struct ext4_allocation_context *ac)
3072{
3073 struct ext4_prealloc_space *pa = ac->ac_pa;
3074 int len;
3075
3076 if (pa && pa->pa_type == MB_INODE_PA) {
3077 len = ac->ac_b_ex.fe_len;
3078 pa->pa_free += len;
3079 }
3080
3081}
3082
3083/*
Alex Tomasc9de5602008-01-29 00:19:52 -05003084 * use blocks preallocated to inode
3085 */
3086static void ext4_mb_use_inode_pa(struct ext4_allocation_context *ac,
3087 struct ext4_prealloc_space *pa)
3088{
Theodore Ts'o53accfa2011-09-09 18:48:51 -04003089 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
Alex Tomasc9de5602008-01-29 00:19:52 -05003090 ext4_fsblk_t start;
3091 ext4_fsblk_t end;
3092 int len;
3093
3094 /* found preallocated blocks, use them */
3095 start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart);
Theodore Ts'o53accfa2011-09-09 18:48:51 -04003096 end = min(pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len),
3097 start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len));
3098 len = EXT4_NUM_B2C(sbi, end - start);
Alex Tomasc9de5602008-01-29 00:19:52 -05003099 ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group,
3100 &ac->ac_b_ex.fe_start);
3101 ac->ac_b_ex.fe_len = len;
3102 ac->ac_status = AC_STATUS_FOUND;
3103 ac->ac_pa = pa;
3104
3105 BUG_ON(start < pa->pa_pstart);
Theodore Ts'o53accfa2011-09-09 18:48:51 -04003106 BUG_ON(end > pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len));
Alex Tomasc9de5602008-01-29 00:19:52 -05003107 BUG_ON(pa->pa_free < len);
3108 pa->pa_free -= len;
3109
Theodore Ts'o6ba495e2009-09-18 13:38:55 -04003110 mb_debug(1, "use %llu/%u from inode pa %p\n", start, len, pa);
Alex Tomasc9de5602008-01-29 00:19:52 -05003111}
3112
3113/*
3114 * use blocks preallocated to locality group
3115 */
3116static void ext4_mb_use_group_pa(struct ext4_allocation_context *ac,
3117 struct ext4_prealloc_space *pa)
3118{
Aneesh Kumar K.V03cddb82008-06-05 20:59:29 -04003119 unsigned int len = ac->ac_o_ex.fe_len;
Aneesh Kumar K.V6be2ded2008-07-23 14:14:05 -04003120
Alex Tomasc9de5602008-01-29 00:19:52 -05003121 ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart,
3122 &ac->ac_b_ex.fe_group,
3123 &ac->ac_b_ex.fe_start);
3124 ac->ac_b_ex.fe_len = len;
3125 ac->ac_status = AC_STATUS_FOUND;
3126 ac->ac_pa = pa;
3127
3128 /* we don't correct pa_pstart or pa_plen here to avoid
Aneesh Kumar K.V26346ff2008-02-10 01:10:04 -05003129 * possible race when the group is being loaded concurrently
Alex Tomasc9de5602008-01-29 00:19:52 -05003130 * instead we correct pa later, after blocks are marked
Aneesh Kumar K.V26346ff2008-02-10 01:10:04 -05003131 * in on-disk bitmap -- see ext4_mb_release_context()
3132 * Other CPUs are prevented from allocating from this pa by lg_mutex
Alex Tomasc9de5602008-01-29 00:19:52 -05003133 */
Theodore Ts'o6ba495e2009-09-18 13:38:55 -04003134 mb_debug(1, "use %u/%u from group pa %p\n", pa->pa_lstart-len, len, pa);
Alex Tomasc9de5602008-01-29 00:19:52 -05003135}
3136
3137/*
Aneesh Kumar K.V5e745b02008-08-18 18:00:57 -04003138 * Return the prealloc space that have minimal distance
3139 * from the goal block. @cpa is the prealloc
3140 * space that is having currently known minimal distance
3141 * from the goal block.
3142 */
3143static struct ext4_prealloc_space *
3144ext4_mb_check_group_pa(ext4_fsblk_t goal_block,
3145 struct ext4_prealloc_space *pa,
3146 struct ext4_prealloc_space *cpa)
3147{
3148 ext4_fsblk_t cur_distance, new_distance;
3149
3150 if (cpa == NULL) {
3151 atomic_inc(&pa->pa_count);
3152 return pa;
3153 }
3154 cur_distance = abs(goal_block - cpa->pa_pstart);
3155 new_distance = abs(goal_block - pa->pa_pstart);
3156
Coly Li5a54b2f2011-02-24 14:10:05 -05003157 if (cur_distance <= new_distance)
Aneesh Kumar K.V5e745b02008-08-18 18:00:57 -04003158 return cpa;
3159
3160 /* drop the previous reference */
3161 atomic_dec(&cpa->pa_count);
3162 atomic_inc(&pa->pa_count);
3163 return pa;
3164}
3165
3166/*
Alex Tomasc9de5602008-01-29 00:19:52 -05003167 * search goal blocks in preallocated space
3168 */
Eric Sandeen4ddfef72008-04-29 08:11:12 -04003169static noinline_for_stack int
3170ext4_mb_use_preallocated(struct ext4_allocation_context *ac)
Alex Tomasc9de5602008-01-29 00:19:52 -05003171{
Theodore Ts'o53accfa2011-09-09 18:48:51 -04003172 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
Aneesh Kumar K.V6be2ded2008-07-23 14:14:05 -04003173 int order, i;
Alex Tomasc9de5602008-01-29 00:19:52 -05003174 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);
3175 struct ext4_locality_group *lg;
Aneesh Kumar K.V5e745b02008-08-18 18:00:57 -04003176 struct ext4_prealloc_space *pa, *cpa = NULL;
3177 ext4_fsblk_t goal_block;
Alex Tomasc9de5602008-01-29 00:19:52 -05003178
3179 /* only data can be preallocated */
3180 if (!(ac->ac_flags & EXT4_MB_HINT_DATA))
3181 return 0;
3182
3183 /* first, try per-file preallocation */
3184 rcu_read_lock();
Aneesh Kumar K.V9a0762c2008-04-17 10:38:59 -04003185 list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) {
Alex Tomasc9de5602008-01-29 00:19:52 -05003186
3187 /* all fields in this condition don't change,
3188 * so we can skip locking for them */
3189 if (ac->ac_o_ex.fe_logical < pa->pa_lstart ||
Theodore Ts'o53accfa2011-09-09 18:48:51 -04003190 ac->ac_o_ex.fe_logical >= (pa->pa_lstart +
3191 EXT4_C2B(sbi, pa->pa_len)))
Alex Tomasc9de5602008-01-29 00:19:52 -05003192 continue;
3193
Eric Sandeenfb0a3872009-09-16 14:45:10 -04003194 /* non-extent files can't have physical blocks past 2^32 */
Dmitry Monakhov12e9b892010-05-16 22:00:00 -04003195 if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) &&
Theodore Ts'o53accfa2011-09-09 18:48:51 -04003196 (pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len) >
3197 EXT4_MAX_BLOCK_FILE_PHYS))
Eric Sandeenfb0a3872009-09-16 14:45:10 -04003198 continue;
3199
Alex Tomasc9de5602008-01-29 00:19:52 -05003200 /* found preallocated blocks, use them */
3201 spin_lock(&pa->pa_lock);
3202 if (pa->pa_deleted == 0 && pa->pa_free) {
3203 atomic_inc(&pa->pa_count);
3204 ext4_mb_use_inode_pa(ac, pa);
3205 spin_unlock(&pa->pa_lock);
3206 ac->ac_criteria = 10;
3207 rcu_read_unlock();
3208 return 1;
3209 }
3210 spin_unlock(&pa->pa_lock);
3211 }
3212 rcu_read_unlock();
3213
3214 /* can we use group allocation? */
3215 if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC))
3216 return 0;
3217
3218 /* inode may have no locality group for some reason */
3219 lg = ac->ac_lg;
3220 if (lg == NULL)
3221 return 0;
Aneesh Kumar K.V6be2ded2008-07-23 14:14:05 -04003222 order = fls(ac->ac_o_ex.fe_len) - 1;
3223 if (order > PREALLOC_TB_SIZE - 1)
3224 /* The max size of hash table is PREALLOC_TB_SIZE */
3225 order = PREALLOC_TB_SIZE - 1;
Alex Tomasc9de5602008-01-29 00:19:52 -05003226
Akinobu Mitabda00de2010-03-03 23:53:25 -05003227 goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex);
Aneesh Kumar K.V5e745b02008-08-18 18:00:57 -04003228 /*
3229 * search for the prealloc space that is having
3230 * minimal distance from the goal block.
3231 */
Aneesh Kumar K.V6be2ded2008-07-23 14:14:05 -04003232 for (i = order; i < PREALLOC_TB_SIZE; i++) {
3233 rcu_read_lock();
3234 list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[i],
3235 pa_inode_list) {
3236 spin_lock(&pa->pa_lock);
3237 if (pa->pa_deleted == 0 &&
3238 pa->pa_free >= ac->ac_o_ex.fe_len) {
Aneesh Kumar K.V5e745b02008-08-18 18:00:57 -04003239
3240 cpa = ext4_mb_check_group_pa(goal_block,
3241 pa, cpa);
Aneesh Kumar K.V6be2ded2008-07-23 14:14:05 -04003242 }
Alex Tomasc9de5602008-01-29 00:19:52 -05003243 spin_unlock(&pa->pa_lock);
Alex Tomasc9de5602008-01-29 00:19:52 -05003244 }
Aneesh Kumar K.V6be2ded2008-07-23 14:14:05 -04003245 rcu_read_unlock();
Alex Tomasc9de5602008-01-29 00:19:52 -05003246 }
Aneesh Kumar K.V5e745b02008-08-18 18:00:57 -04003247 if (cpa) {
3248 ext4_mb_use_group_pa(ac, cpa);
3249 ac->ac_criteria = 20;
3250 return 1;
3251 }
Alex Tomasc9de5602008-01-29 00:19:52 -05003252 return 0;
3253}
3254
3255/*
Aneesh Kumar K.V7a2fcbf2009-01-05 21:36:55 -05003256 * the function goes through all block freed in the group
3257 * but not yet committed and marks them used in in-core bitmap.
3258 * buddy must be generated from this bitmap
Aneesh Kumar K.V955ce5f2009-05-02 20:35:09 -04003259 * Need to be called with the ext4 group lock held
Aneesh Kumar K.V7a2fcbf2009-01-05 21:36:55 -05003260 */
3261static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap,
3262 ext4_group_t group)
3263{
3264 struct rb_node *n;
3265 struct ext4_group_info *grp;
3266 struct ext4_free_data *entry;
3267
3268 grp = ext4_get_group_info(sb, group);
3269 n = rb_first(&(grp->bb_free_root));
3270
3271 while (n) {
Bobi Jam18aadd42012-02-20 17:53:02 -05003272 entry = rb_entry(n, struct ext4_free_data, efd_node);
3273 ext4_set_bits(bitmap, entry->efd_start_cluster, entry->efd_count);
Aneesh Kumar K.V7a2fcbf2009-01-05 21:36:55 -05003274 n = rb_next(n);
3275 }
3276 return;
3277}
3278
3279/*
Alex Tomasc9de5602008-01-29 00:19:52 -05003280 * the function goes through all preallocation in this group and marks them
3281 * used in in-core bitmap. buddy must be generated from this bitmap
Aneesh Kumar K.V955ce5f2009-05-02 20:35:09 -04003282 * Need to be called with ext4 group lock held
Alex Tomasc9de5602008-01-29 00:19:52 -05003283 */
Eric Sandeen089ceec2009-07-05 22:17:31 -04003284static noinline_for_stack
3285void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap,
Alex Tomasc9de5602008-01-29 00:19:52 -05003286 ext4_group_t group)
3287{
3288 struct ext4_group_info *grp = ext4_get_group_info(sb, group);
3289 struct ext4_prealloc_space *pa;
3290 struct list_head *cur;
3291 ext4_group_t groupnr;
3292 ext4_grpblk_t start;
3293 int preallocated = 0;
Alex Tomasc9de5602008-01-29 00:19:52 -05003294 int len;
3295
3296 /* all form of preallocation discards first load group,
3297 * so the only competing code is preallocation use.
3298 * we don't need any locking here
3299 * notice we do NOT ignore preallocations with pa_deleted
3300 * otherwise we could leave used blocks available for
3301 * allocation in buddy when concurrent ext4_mb_put_pa()
3302 * is dropping preallocation
3303 */
3304 list_for_each(cur, &grp->bb_prealloc_list) {
3305 pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list);
3306 spin_lock(&pa->pa_lock);
3307 ext4_get_group_no_and_offset(sb, pa->pa_pstart,
3308 &groupnr, &start);
3309 len = pa->pa_len;
3310 spin_unlock(&pa->pa_lock);
3311 if (unlikely(len == 0))
3312 continue;
3313 BUG_ON(groupnr != group);
Yongqiang Yangc3e94d12011-07-26 22:05:53 -04003314 ext4_set_bits(bitmap, start, len);
Alex Tomasc9de5602008-01-29 00:19:52 -05003315 preallocated += len;
Alex Tomasc9de5602008-01-29 00:19:52 -05003316 }
Theodore Ts'o6ba495e2009-09-18 13:38:55 -04003317 mb_debug(1, "prellocated %u for group %u\n", preallocated, group);
Alex Tomasc9de5602008-01-29 00:19:52 -05003318}
3319
3320static void ext4_mb_pa_callback(struct rcu_head *head)
3321{
3322 struct ext4_prealloc_space *pa;
3323 pa = container_of(head, struct ext4_prealloc_space, u.pa_rcu);
3324 kmem_cache_free(ext4_pspace_cachep, pa);
3325}
3326
3327/*
3328 * drops a reference to preallocated space descriptor
3329 * if this was the last reference and the space is consumed
3330 */
3331static void ext4_mb_put_pa(struct ext4_allocation_context *ac,
3332 struct super_block *sb, struct ext4_prealloc_space *pa)
3333{
Theodore Ts'oa9df9a42009-01-05 22:18:16 -05003334 ext4_group_t grp;
Eric Sandeend33a1972009-03-16 23:25:40 -04003335 ext4_fsblk_t grp_blk;
Alex Tomasc9de5602008-01-29 00:19:52 -05003336
3337 if (!atomic_dec_and_test(&pa->pa_count) || pa->pa_free != 0)
3338 return;
3339
3340 /* in this short window concurrent discard can set pa_deleted */
3341 spin_lock(&pa->pa_lock);
3342 if (pa->pa_deleted == 1) {
3343 spin_unlock(&pa->pa_lock);
3344 return;
3345 }
3346
3347 pa->pa_deleted = 1;
3348 spin_unlock(&pa->pa_lock);
3349
Eric Sandeend33a1972009-03-16 23:25:40 -04003350 grp_blk = pa->pa_pstart;
Theodore Ts'o60e66792010-05-17 07:00:00 -04003351 /*
Aneesh Kumar K.Vcc0fb9a2009-03-27 17:16:58 -04003352 * If doing group-based preallocation, pa_pstart may be in the
3353 * next group when pa is used up
3354 */
3355 if (pa->pa_type == MB_GROUP_PA)
Eric Sandeend33a1972009-03-16 23:25:40 -04003356 grp_blk--;
3357
3358 ext4_get_group_no_and_offset(sb, grp_blk, &grp, NULL);
Alex Tomasc9de5602008-01-29 00:19:52 -05003359
3360 /*
3361 * possible race:
3362 *
3363 * P1 (buddy init) P2 (regular allocation)
3364 * find block B in PA
3365 * copy on-disk bitmap to buddy
3366 * mark B in on-disk bitmap
3367 * drop PA from group
3368 * mark all PAs in buddy
3369 *
3370 * thus, P1 initializes buddy with B available. to prevent this
3371 * we make "copy" and "mark all PAs" atomic and serialize "drop PA"
3372 * against that pair
3373 */
3374 ext4_lock_group(sb, grp);
3375 list_del(&pa->pa_group_list);
3376 ext4_unlock_group(sb, grp);
3377
3378 spin_lock(pa->pa_obj_lock);
3379 list_del_rcu(&pa->pa_inode_list);
3380 spin_unlock(pa->pa_obj_lock);
3381
3382 call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback);
3383}
3384
3385/*
3386 * creates new preallocated space for given inode
3387 */
Eric Sandeen4ddfef72008-04-29 08:11:12 -04003388static noinline_for_stack int
3389ext4_mb_new_inode_pa(struct ext4_allocation_context *ac)
Alex Tomasc9de5602008-01-29 00:19:52 -05003390{
3391 struct super_block *sb = ac->ac_sb;
Theodore Ts'o53accfa2011-09-09 18:48:51 -04003392 struct ext4_sb_info *sbi = EXT4_SB(sb);
Alex Tomasc9de5602008-01-29 00:19:52 -05003393 struct ext4_prealloc_space *pa;
3394 struct ext4_group_info *grp;
3395 struct ext4_inode_info *ei;
3396
3397 /* preallocate only when found space is larger then requested */
3398 BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len);
3399 BUG_ON(ac->ac_status != AC_STATUS_FOUND);
3400 BUG_ON(!S_ISREG(ac->ac_inode->i_mode));
3401
3402 pa = kmem_cache_alloc(ext4_pspace_cachep, GFP_NOFS);
3403 if (pa == NULL)
3404 return -ENOMEM;
3405
3406 if (ac->ac_b_ex.fe_len < ac->ac_g_ex.fe_len) {
3407 int winl;
3408 int wins;
3409 int win;
3410 int offs;
3411
3412 /* we can't allocate as much as normalizer wants.
3413 * so, found space must get proper lstart
3414 * to cover original request */
3415 BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical);
3416 BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len);
3417
3418 /* we're limited by original request in that
3419 * logical block must be covered any way
3420 * winl is window we can move our chunk within */
3421 winl = ac->ac_o_ex.fe_logical - ac->ac_g_ex.fe_logical;
3422
3423 /* also, we should cover whole original request */
Theodore Ts'o53accfa2011-09-09 18:48:51 -04003424 wins = EXT4_C2B(sbi, ac->ac_b_ex.fe_len - ac->ac_o_ex.fe_len);
Alex Tomasc9de5602008-01-29 00:19:52 -05003425
3426 /* the smallest one defines real window */
3427 win = min(winl, wins);
3428
Theodore Ts'o53accfa2011-09-09 18:48:51 -04003429 offs = ac->ac_o_ex.fe_logical %
3430 EXT4_C2B(sbi, ac->ac_b_ex.fe_len);
Alex Tomasc9de5602008-01-29 00:19:52 -05003431 if (offs && offs < win)
3432 win = offs;
3433
Theodore Ts'o53accfa2011-09-09 18:48:51 -04003434 ac->ac_b_ex.fe_logical = ac->ac_o_ex.fe_logical -
3435 EXT4_B2C(sbi, win);
Alex Tomasc9de5602008-01-29 00:19:52 -05003436 BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical);
3437 BUG_ON(ac->ac_o_ex.fe_len > ac->ac_b_ex.fe_len);
3438 }
3439
3440 /* preallocation can change ac_b_ex, thus we store actually
3441 * allocated blocks for history */
3442 ac->ac_f_ex = ac->ac_b_ex;
3443
3444 pa->pa_lstart = ac->ac_b_ex.fe_logical;
3445 pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);
3446 pa->pa_len = ac->ac_b_ex.fe_len;
3447 pa->pa_free = pa->pa_len;
3448 atomic_set(&pa->pa_count, 1);
3449 spin_lock_init(&pa->pa_lock);
Aneesh Kumar K.Vd794bf82009-02-14 10:31:16 -05003450 INIT_LIST_HEAD(&pa->pa_inode_list);
3451 INIT_LIST_HEAD(&pa->pa_group_list);
Alex Tomasc9de5602008-01-29 00:19:52 -05003452 pa->pa_deleted = 0;
Aneesh Kumar K.Vcc0fb9a2009-03-27 17:16:58 -04003453 pa->pa_type = MB_INODE_PA;
Alex Tomasc9de5602008-01-29 00:19:52 -05003454
Theodore Ts'o6ba495e2009-09-18 13:38:55 -04003455 mb_debug(1, "new inode pa %p: %llu/%u for %u\n", pa,
Alex Tomasc9de5602008-01-29 00:19:52 -05003456 pa->pa_pstart, pa->pa_len, pa->pa_lstart);
Theodore Ts'o9bffad12009-06-17 11:48:11 -04003457 trace_ext4_mb_new_inode_pa(ac, pa);
Alex Tomasc9de5602008-01-29 00:19:52 -05003458
3459 ext4_mb_use_inode_pa(ac, pa);
Theodore Ts'o53accfa2011-09-09 18:48:51 -04003460 atomic_add(pa->pa_free, &sbi->s_mb_preallocated);
Alex Tomasc9de5602008-01-29 00:19:52 -05003461
3462 ei = EXT4_I(ac->ac_inode);
3463 grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);
3464
3465 pa->pa_obj_lock = &ei->i_prealloc_lock;
3466 pa->pa_inode = ac->ac_inode;
3467
3468 ext4_lock_group(sb, ac->ac_b_ex.fe_group);
3469 list_add(&pa->pa_group_list, &grp->bb_prealloc_list);
3470 ext4_unlock_group(sb, ac->ac_b_ex.fe_group);
3471
3472 spin_lock(pa->pa_obj_lock);
3473 list_add_rcu(&pa->pa_inode_list, &ei->i_prealloc_list);
3474 spin_unlock(pa->pa_obj_lock);
3475
3476 return 0;
3477}
3478
3479/*
3480 * creates new preallocated space for locality group inodes belongs to
3481 */
Eric Sandeen4ddfef72008-04-29 08:11:12 -04003482static noinline_for_stack int
3483ext4_mb_new_group_pa(struct ext4_allocation_context *ac)
Alex Tomasc9de5602008-01-29 00:19:52 -05003484{
3485 struct super_block *sb = ac->ac_sb;
3486 struct ext4_locality_group *lg;
3487 struct ext4_prealloc_space *pa;
3488 struct ext4_group_info *grp;
3489
3490 /* preallocate only when found space is larger then requested */
3491 BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len);
3492 BUG_ON(ac->ac_status != AC_STATUS_FOUND);
3493 BUG_ON(!S_ISREG(ac->ac_inode->i_mode));
3494
3495 BUG_ON(ext4_pspace_cachep == NULL);
3496 pa = kmem_cache_alloc(ext4_pspace_cachep, GFP_NOFS);
3497 if (pa == NULL)
3498 return -ENOMEM;
3499
3500 /* preallocation can change ac_b_ex, thus we store actually
3501 * allocated blocks for history */
3502 ac->ac_f_ex = ac->ac_b_ex;
3503
3504 pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);
3505 pa->pa_lstart = pa->pa_pstart;
3506 pa->pa_len = ac->ac_b_ex.fe_len;
3507 pa->pa_free = pa->pa_len;
3508 atomic_set(&pa->pa_count, 1);
3509 spin_lock_init(&pa->pa_lock);
Aneesh Kumar K.V6be2ded2008-07-23 14:14:05 -04003510 INIT_LIST_HEAD(&pa->pa_inode_list);
Aneesh Kumar K.Vd794bf82009-02-14 10:31:16 -05003511 INIT_LIST_HEAD(&pa->pa_group_list);
Alex Tomasc9de5602008-01-29 00:19:52 -05003512 pa->pa_deleted = 0;
Aneesh Kumar K.Vcc0fb9a2009-03-27 17:16:58 -04003513 pa->pa_type = MB_GROUP_PA;
Alex Tomasc9de5602008-01-29 00:19:52 -05003514
Theodore Ts'o6ba495e2009-09-18 13:38:55 -04003515 mb_debug(1, "new group pa %p: %llu/%u for %u\n", pa,
Theodore Ts'o9bffad12009-06-17 11:48:11 -04003516 pa->pa_pstart, pa->pa_len, pa->pa_lstart);
3517 trace_ext4_mb_new_group_pa(ac, pa);
Alex Tomasc9de5602008-01-29 00:19:52 -05003518
3519 ext4_mb_use_group_pa(ac, pa);
3520 atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated);
3521
3522 grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);
3523 lg = ac->ac_lg;
3524 BUG_ON(lg == NULL);
3525
3526 pa->pa_obj_lock = &lg->lg_prealloc_lock;
3527 pa->pa_inode = NULL;
3528
3529 ext4_lock_group(sb, ac->ac_b_ex.fe_group);
3530 list_add(&pa->pa_group_list, &grp->bb_prealloc_list);
3531 ext4_unlock_group(sb, ac->ac_b_ex.fe_group);
3532
Aneesh Kumar K.V6be2ded2008-07-23 14:14:05 -04003533 /*
3534 * We will later add the new pa to the right bucket
3535 * after updating the pa_free in ext4_mb_release_context
3536 */
Alex Tomasc9de5602008-01-29 00:19:52 -05003537 return 0;
3538}
3539
3540static int ext4_mb_new_preallocation(struct ext4_allocation_context *ac)
3541{
3542 int err;
3543
3544 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)
3545 err = ext4_mb_new_group_pa(ac);
3546 else
3547 err = ext4_mb_new_inode_pa(ac);
3548 return err;
3549}
3550
3551/*
3552 * finds all unused blocks in on-disk bitmap, frees them in
3553 * in-core bitmap and buddy.
3554 * @pa must be unlinked from inode and group lists, so that
3555 * nobody else can find/use it.
3556 * the caller MUST hold group/inode locks.
3557 * TODO: optimize the case when there are no in-core structures yet
3558 */
Eric Sandeen4ddfef72008-04-29 08:11:12 -04003559static noinline_for_stack int
3560ext4_mb_release_inode_pa(struct ext4_buddy *e4b, struct buffer_head *bitmap_bh,
Eric Sandeen3e1e5f52010-10-27 21:30:07 -04003561 struct ext4_prealloc_space *pa)
Alex Tomasc9de5602008-01-29 00:19:52 -05003562{
Alex Tomasc9de5602008-01-29 00:19:52 -05003563 struct super_block *sb = e4b->bd_sb;
3564 struct ext4_sb_info *sbi = EXT4_SB(sb);
Theodore Ts'o498e5f22008-11-05 00:14:04 -05003565 unsigned int end;
3566 unsigned int next;
Alex Tomasc9de5602008-01-29 00:19:52 -05003567 ext4_group_t group;
3568 ext4_grpblk_t bit;
Theodore Ts'oba80b102009-01-03 20:03:21 -05003569 unsigned long long grp_blk_start;
Alex Tomasc9de5602008-01-29 00:19:52 -05003570 int err = 0;
3571 int free = 0;
3572
3573 BUG_ON(pa->pa_deleted == 0);
3574 ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit);
Theodore Ts'o53accfa2011-09-09 18:48:51 -04003575 grp_blk_start = pa->pa_pstart - EXT4_C2B(sbi, bit);
Alex Tomasc9de5602008-01-29 00:19:52 -05003576 BUG_ON(group != e4b->bd_group && pa->pa_len != 0);
3577 end = bit + pa->pa_len;
3578
Alex Tomasc9de5602008-01-29 00:19:52 -05003579 while (bit < end) {
Aneesh Kumar K.Vffad0a42008-02-23 01:38:34 -05003580 bit = mb_find_next_zero_bit(bitmap_bh->b_data, end, bit);
Alex Tomasc9de5602008-01-29 00:19:52 -05003581 if (bit >= end)
3582 break;
Aneesh Kumar K.Vffad0a42008-02-23 01:38:34 -05003583 next = mb_find_next_bit(bitmap_bh->b_data, end, bit);
Theodore Ts'o6ba495e2009-09-18 13:38:55 -04003584 mb_debug(1, " free preallocated %u/%u in group %u\n",
Andi Kleen5a0790c2010-06-14 13:28:03 -04003585 (unsigned) ext4_group_first_block_no(sb, group) + bit,
3586 (unsigned) next - bit, (unsigned) group);
Alex Tomasc9de5602008-01-29 00:19:52 -05003587 free += next - bit;
3588
Eric Sandeen3e1e5f52010-10-27 21:30:07 -04003589 trace_ext4_mballoc_discard(sb, NULL, group, bit, next - bit);
Theodore Ts'o53accfa2011-09-09 18:48:51 -04003590 trace_ext4_mb_release_inode_pa(pa, (grp_blk_start +
3591 EXT4_C2B(sbi, bit)),
Lukas Czernera9c667f2011-06-06 09:51:52 -04003592 next - bit);
Alex Tomasc9de5602008-01-29 00:19:52 -05003593 mb_free_blocks(pa->pa_inode, e4b, bit, next - bit);
3594 bit = next + 1;
3595 }
3596 if (free != pa->pa_free) {
Theodore Ts'o9d8b9ec2011-08-01 17:41:35 -04003597 ext4_msg(e4b->bd_sb, KERN_CRIT,
3598 "pa %p: logic %lu, phys. %lu, len %lu",
3599 pa, (unsigned long) pa->pa_lstart,
3600 (unsigned long) pa->pa_pstart,
3601 (unsigned long) pa->pa_len);
Theodore Ts'oe29136f2010-06-29 12:54:28 -04003602 ext4_grp_locked_error(sb, group, 0, 0, "free %u, pa_free %u",
Aneesh Kumar K.V5d1b1b32009-01-05 22:19:52 -05003603 free, pa->pa_free);
Aneesh Kumar K.Ve56eb652008-02-15 13:48:21 -05003604 /*
3605 * pa is already deleted so we use the value obtained
3606 * from the bitmap and continue.
3607 */
Alex Tomasc9de5602008-01-29 00:19:52 -05003608 }
Alex Tomasc9de5602008-01-29 00:19:52 -05003609 atomic_add(free, &sbi->s_mb_discarded);
3610
3611 return err;
3612}
3613
Eric Sandeen4ddfef72008-04-29 08:11:12 -04003614static noinline_for_stack int
3615ext4_mb_release_group_pa(struct ext4_buddy *e4b,
Eric Sandeen3e1e5f52010-10-27 21:30:07 -04003616 struct ext4_prealloc_space *pa)
Alex Tomasc9de5602008-01-29 00:19:52 -05003617{
Alex Tomasc9de5602008-01-29 00:19:52 -05003618 struct super_block *sb = e4b->bd_sb;
3619 ext4_group_t group;
3620 ext4_grpblk_t bit;
3621
Yongqiang Yang60e07cf2011-12-18 15:49:54 -05003622 trace_ext4_mb_release_group_pa(sb, pa);
Alex Tomasc9de5602008-01-29 00:19:52 -05003623 BUG_ON(pa->pa_deleted == 0);
3624 ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit);
3625 BUG_ON(group != e4b->bd_group && pa->pa_len != 0);
3626 mb_free_blocks(pa->pa_inode, e4b, bit, pa->pa_len);
3627 atomic_add(pa->pa_len, &EXT4_SB(sb)->s_mb_discarded);
Eric Sandeen3e1e5f52010-10-27 21:30:07 -04003628 trace_ext4_mballoc_discard(sb, NULL, group, bit, pa->pa_len);
Alex Tomasc9de5602008-01-29 00:19:52 -05003629
3630 return 0;
3631}
3632
3633/*
3634 * releases all preallocations in given group
3635 *
3636 * first, we need to decide discard policy:
3637 * - when do we discard
3638 * 1) ENOSPC
3639 * - how many do we discard
3640 * 1) how many requested
3641 */
Eric Sandeen4ddfef72008-04-29 08:11:12 -04003642static noinline_for_stack int
3643ext4_mb_discard_group_preallocations(struct super_block *sb,
Alex Tomasc9de5602008-01-29 00:19:52 -05003644 ext4_group_t group, int needed)
3645{
3646 struct ext4_group_info *grp = ext4_get_group_info(sb, group);
3647 struct buffer_head *bitmap_bh = NULL;
3648 struct ext4_prealloc_space *pa, *tmp;
3649 struct list_head list;
3650 struct ext4_buddy e4b;
3651 int err;
3652 int busy = 0;
3653 int free = 0;
3654
Theodore Ts'o6ba495e2009-09-18 13:38:55 -04003655 mb_debug(1, "discard preallocation for group %u\n", group);
Alex Tomasc9de5602008-01-29 00:19:52 -05003656
3657 if (list_empty(&grp->bb_prealloc_list))
3658 return 0;
3659
Theodore Ts'o574ca172008-07-11 19:27:31 -04003660 bitmap_bh = ext4_read_block_bitmap(sb, group);
Alex Tomasc9de5602008-01-29 00:19:52 -05003661 if (bitmap_bh == NULL) {
Eric Sandeen12062dd2010-02-15 14:19:27 -05003662 ext4_error(sb, "Error reading block bitmap for %u", group);
Aneesh Kumar K.Vce89f462008-07-23 14:09:29 -04003663 return 0;
Alex Tomasc9de5602008-01-29 00:19:52 -05003664 }
3665
3666 err = ext4_mb_load_buddy(sb, group, &e4b);
Aneesh Kumar K.Vce89f462008-07-23 14:09:29 -04003667 if (err) {
Eric Sandeen12062dd2010-02-15 14:19:27 -05003668 ext4_error(sb, "Error loading buddy information for %u", group);
Aneesh Kumar K.Vce89f462008-07-23 14:09:29 -04003669 put_bh(bitmap_bh);
3670 return 0;
3671 }
Alex Tomasc9de5602008-01-29 00:19:52 -05003672
3673 if (needed == 0)
Theodore Ts'o7137d7a2011-09-09 18:38:51 -04003674 needed = EXT4_CLUSTERS_PER_GROUP(sb) + 1;
Alex Tomasc9de5602008-01-29 00:19:52 -05003675
Alex Tomasc9de5602008-01-29 00:19:52 -05003676 INIT_LIST_HEAD(&list);
Alex Tomasc9de5602008-01-29 00:19:52 -05003677repeat:
3678 ext4_lock_group(sb, group);
3679 list_for_each_entry_safe(pa, tmp,
3680 &grp->bb_prealloc_list, pa_group_list) {
3681 spin_lock(&pa->pa_lock);
3682 if (atomic_read(&pa->pa_count)) {
3683 spin_unlock(&pa->pa_lock);
3684 busy = 1;
3685 continue;
3686 }
3687 if (pa->pa_deleted) {
3688 spin_unlock(&pa->pa_lock);
3689 continue;
3690 }
3691
3692 /* seems this one can be freed ... */
3693 pa->pa_deleted = 1;
3694
3695 /* we can trust pa_free ... */
3696 free += pa->pa_free;
3697
3698 spin_unlock(&pa->pa_lock);
3699
3700 list_del(&pa->pa_group_list);
3701 list_add(&pa->u.pa_tmp_list, &list);
3702 }
3703
3704 /* if we still need more blocks and some PAs were used, try again */
3705 if (free < needed && busy) {
3706 busy = 0;
3707 ext4_unlock_group(sb, group);
3708 /*
3709 * Yield the CPU here so that we don't get soft lockup
3710 * in non preempt case.
3711 */
3712 yield();
3713 goto repeat;
3714 }
3715
3716 /* found anything to free? */
3717 if (list_empty(&list)) {
3718 BUG_ON(free != 0);
3719 goto out;
3720 }
3721
3722 /* now free all selected PAs */
3723 list_for_each_entry_safe(pa, tmp, &list, u.pa_tmp_list) {
3724
3725 /* remove from object (inode or locality group) */
3726 spin_lock(pa->pa_obj_lock);
3727 list_del_rcu(&pa->pa_inode_list);
3728 spin_unlock(pa->pa_obj_lock);
3729
Aneesh Kumar K.Vcc0fb9a2009-03-27 17:16:58 -04003730 if (pa->pa_type == MB_GROUP_PA)
Eric Sandeen3e1e5f52010-10-27 21:30:07 -04003731 ext4_mb_release_group_pa(&e4b, pa);
Alex Tomasc9de5602008-01-29 00:19:52 -05003732 else
Eric Sandeen3e1e5f52010-10-27 21:30:07 -04003733 ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa);
Alex Tomasc9de5602008-01-29 00:19:52 -05003734
3735 list_del(&pa->u.pa_tmp_list);
3736 call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback);
3737 }
3738
3739out:
3740 ext4_unlock_group(sb, group);
Jing Zhange39e07f2010-05-14 00:00:00 -04003741 ext4_mb_unload_buddy(&e4b);
Alex Tomasc9de5602008-01-29 00:19:52 -05003742 put_bh(bitmap_bh);
3743 return free;
3744}
3745
3746/*
3747 * releases all non-used preallocated blocks for given inode
3748 *
3749 * It's important to discard preallocations under i_data_sem
3750 * We don't want another block to be served from the prealloc
3751 * space when we are discarding the inode prealloc space.
3752 *
3753 * FIXME!! Make sure it is valid at all the call sites
3754 */
Theodore Ts'oc2ea3fd2008-10-10 09:40:52 -04003755void ext4_discard_preallocations(struct inode *inode)
Alex Tomasc9de5602008-01-29 00:19:52 -05003756{
3757 struct ext4_inode_info *ei = EXT4_I(inode);
3758 struct super_block *sb = inode->i_sb;
3759 struct buffer_head *bitmap_bh = NULL;
3760 struct ext4_prealloc_space *pa, *tmp;
3761 ext4_group_t group = 0;
3762 struct list_head list;
3763 struct ext4_buddy e4b;
3764 int err;
3765
Theodore Ts'oc2ea3fd2008-10-10 09:40:52 -04003766 if (!S_ISREG(inode->i_mode)) {
Alex Tomasc9de5602008-01-29 00:19:52 -05003767 /*BUG_ON(!list_empty(&ei->i_prealloc_list));*/
3768 return;
3769 }
3770
Theodore Ts'o6ba495e2009-09-18 13:38:55 -04003771 mb_debug(1, "discard preallocation for inode %lu\n", inode->i_ino);
Theodore Ts'o9bffad12009-06-17 11:48:11 -04003772 trace_ext4_discard_preallocations(inode);
Alex Tomasc9de5602008-01-29 00:19:52 -05003773
3774 INIT_LIST_HEAD(&list);
3775
3776repeat:
3777 /* first, collect all pa's in the inode */
3778 spin_lock(&ei->i_prealloc_lock);
3779 while (!list_empty(&ei->i_prealloc_list)) {
3780 pa = list_entry(ei->i_prealloc_list.next,
3781 struct ext4_prealloc_space, pa_inode_list);
3782 BUG_ON(pa->pa_obj_lock != &ei->i_prealloc_lock);
3783 spin_lock(&pa->pa_lock);
3784 if (atomic_read(&pa->pa_count)) {
3785 /* this shouldn't happen often - nobody should
3786 * use preallocation while we're discarding it */
3787 spin_unlock(&pa->pa_lock);
3788 spin_unlock(&ei->i_prealloc_lock);
Theodore Ts'o9d8b9ec2011-08-01 17:41:35 -04003789 ext4_msg(sb, KERN_ERR,
3790 "uh-oh! used pa while discarding");
Alex Tomasc9de5602008-01-29 00:19:52 -05003791 WARN_ON(1);
3792 schedule_timeout_uninterruptible(HZ);
3793 goto repeat;
3794
3795 }
3796 if (pa->pa_deleted == 0) {
3797 pa->pa_deleted = 1;
3798 spin_unlock(&pa->pa_lock);
3799 list_del_rcu(&pa->pa_inode_list);
3800 list_add(&pa->u.pa_tmp_list, &list);
3801 continue;
3802 }
3803
3804 /* someone is deleting pa right now */
3805 spin_unlock(&pa->pa_lock);
3806 spin_unlock(&ei->i_prealloc_lock);
3807
3808 /* we have to wait here because pa_deleted
3809 * doesn't mean pa is already unlinked from
3810 * the list. as we might be called from
3811 * ->clear_inode() the inode will get freed
3812 * and concurrent thread which is unlinking
3813 * pa from inode's list may access already
3814 * freed memory, bad-bad-bad */
3815
3816 /* XXX: if this happens too often, we can
3817 * add a flag to force wait only in case
3818 * of ->clear_inode(), but not in case of
3819 * regular truncate */
3820 schedule_timeout_uninterruptible(HZ);
3821 goto repeat;
3822 }
3823 spin_unlock(&ei->i_prealloc_lock);
3824
3825 list_for_each_entry_safe(pa, tmp, &list, u.pa_tmp_list) {
Aneesh Kumar K.Vcc0fb9a2009-03-27 17:16:58 -04003826 BUG_ON(pa->pa_type != MB_INODE_PA);
Alex Tomasc9de5602008-01-29 00:19:52 -05003827 ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, NULL);
3828
3829 err = ext4_mb_load_buddy(sb, group, &e4b);
Aneesh Kumar K.Vce89f462008-07-23 14:09:29 -04003830 if (err) {
Eric Sandeen12062dd2010-02-15 14:19:27 -05003831 ext4_error(sb, "Error loading buddy information for %u",
3832 group);
Aneesh Kumar K.Vce89f462008-07-23 14:09:29 -04003833 continue;
3834 }
Alex Tomasc9de5602008-01-29 00:19:52 -05003835
Theodore Ts'o574ca172008-07-11 19:27:31 -04003836 bitmap_bh = ext4_read_block_bitmap(sb, group);
Alex Tomasc9de5602008-01-29 00:19:52 -05003837 if (bitmap_bh == NULL) {
Eric Sandeen12062dd2010-02-15 14:19:27 -05003838 ext4_error(sb, "Error reading block bitmap for %u",
3839 group);
Jing Zhange39e07f2010-05-14 00:00:00 -04003840 ext4_mb_unload_buddy(&e4b);
Aneesh Kumar K.Vce89f462008-07-23 14:09:29 -04003841 continue;
Alex Tomasc9de5602008-01-29 00:19:52 -05003842 }
3843
3844 ext4_lock_group(sb, group);
3845 list_del(&pa->pa_group_list);
Eric Sandeen3e1e5f52010-10-27 21:30:07 -04003846 ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa);
Alex Tomasc9de5602008-01-29 00:19:52 -05003847 ext4_unlock_group(sb, group);
3848
Jing Zhange39e07f2010-05-14 00:00:00 -04003849 ext4_mb_unload_buddy(&e4b);
Alex Tomasc9de5602008-01-29 00:19:52 -05003850 put_bh(bitmap_bh);
3851
3852 list_del(&pa->u.pa_tmp_list);
3853 call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback);
3854 }
3855}
3856
Theodore Ts'o6ba495e2009-09-18 13:38:55 -04003857#ifdef CONFIG_EXT4_DEBUG
Alex Tomasc9de5602008-01-29 00:19:52 -05003858static void ext4_mb_show_ac(struct ext4_allocation_context *ac)
3859{
3860 struct super_block *sb = ac->ac_sb;
Theodore Ts'o8df96752009-05-01 08:50:38 -04003861 ext4_group_t ngroups, i;
Alex Tomasc9de5602008-01-29 00:19:52 -05003862
Theodore Ts'o4dd89fc2011-02-27 17:23:47 -05003863 if (!mb_enable_debug ||
3864 (EXT4_SB(sb)->s_mount_flags & EXT4_MF_FS_ABORTED))
Eric Sandeene3570632010-07-27 11:56:08 -04003865 return;
3866
Joe Perches7f6a11e2012-03-19 23:09:43 -04003867 ext4_msg(ac->ac_sb, KERN_ERR, "Can't allocate:"
Theodore Ts'o9d8b9ec2011-08-01 17:41:35 -04003868 " Allocation context details:");
Joe Perches7f6a11e2012-03-19 23:09:43 -04003869 ext4_msg(ac->ac_sb, KERN_ERR, "status %d flags %d",
Alex Tomasc9de5602008-01-29 00:19:52 -05003870 ac->ac_status, ac->ac_flags);
Joe Perches7f6a11e2012-03-19 23:09:43 -04003871 ext4_msg(ac->ac_sb, KERN_ERR, "orig %lu/%lu/%lu@%lu, "
Theodore Ts'o9d8b9ec2011-08-01 17:41:35 -04003872 "goal %lu/%lu/%lu@%lu, "
3873 "best %lu/%lu/%lu@%lu cr %d",
Alex Tomasc9de5602008-01-29 00:19:52 -05003874 (unsigned long)ac->ac_o_ex.fe_group,
3875 (unsigned long)ac->ac_o_ex.fe_start,
3876 (unsigned long)ac->ac_o_ex.fe_len,
3877 (unsigned long)ac->ac_o_ex.fe_logical,
3878 (unsigned long)ac->ac_g_ex.fe_group,
3879 (unsigned long)ac->ac_g_ex.fe_start,
3880 (unsigned long)ac->ac_g_ex.fe_len,
3881 (unsigned long)ac->ac_g_ex.fe_logical,
3882 (unsigned long)ac->ac_b_ex.fe_group,
3883 (unsigned long)ac->ac_b_ex.fe_start,
3884 (unsigned long)ac->ac_b_ex.fe_len,
3885 (unsigned long)ac->ac_b_ex.fe_logical,
3886 (int)ac->ac_criteria);
Joe Perches7f6a11e2012-03-19 23:09:43 -04003887 ext4_msg(ac->ac_sb, KERN_ERR, "%lu scanned, %d found",
Theodore Ts'o9d8b9ec2011-08-01 17:41:35 -04003888 ac->ac_ex_scanned, ac->ac_found);
Joe Perches7f6a11e2012-03-19 23:09:43 -04003889 ext4_msg(ac->ac_sb, KERN_ERR, "groups: ");
Theodore Ts'o8df96752009-05-01 08:50:38 -04003890 ngroups = ext4_get_groups_count(sb);
3891 for (i = 0; i < ngroups; i++) {
Alex Tomasc9de5602008-01-29 00:19:52 -05003892 struct ext4_group_info *grp = ext4_get_group_info(sb, i);
3893 struct ext4_prealloc_space *pa;
3894 ext4_grpblk_t start;
3895 struct list_head *cur;
3896 ext4_lock_group(sb, i);
3897 list_for_each(cur, &grp->bb_prealloc_list) {
3898 pa = list_entry(cur, struct ext4_prealloc_space,
3899 pa_group_list);
3900 spin_lock(&pa->pa_lock);
3901 ext4_get_group_no_and_offset(sb, pa->pa_pstart,
3902 NULL, &start);
3903 spin_unlock(&pa->pa_lock);
Akira Fujita1c718502009-07-05 23:04:36 -04003904 printk(KERN_ERR "PA:%u:%d:%u \n", i,
3905 start, pa->pa_len);
Alex Tomasc9de5602008-01-29 00:19:52 -05003906 }
Solofo Ramangalahy60bd63d2008-04-29 21:59:59 -04003907 ext4_unlock_group(sb, i);
Alex Tomasc9de5602008-01-29 00:19:52 -05003908
3909 if (grp->bb_free == 0)
3910 continue;
Akira Fujita1c718502009-07-05 23:04:36 -04003911 printk(KERN_ERR "%u: %d/%d \n",
Alex Tomasc9de5602008-01-29 00:19:52 -05003912 i, grp->bb_free, grp->bb_fragments);
3913 }
3914 printk(KERN_ERR "\n");
3915}
3916#else
3917static inline void ext4_mb_show_ac(struct ext4_allocation_context *ac)
3918{
3919 return;
3920}
3921#endif
3922
3923/*
3924 * We use locality group preallocation for small size file. The size of the
3925 * file is determined by the current size or the resulting size after
3926 * allocation which ever is larger
3927 *
Theodore Ts'ob713a5e2009-03-31 09:11:14 -04003928 * One can tune this size via /sys/fs/ext4/<partition>/mb_stream_req
Alex Tomasc9de5602008-01-29 00:19:52 -05003929 */
3930static void ext4_mb_group_or_file(struct ext4_allocation_context *ac)
3931{
3932 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
3933 int bsbits = ac->ac_sb->s_blocksize_bits;
3934 loff_t size, isize;
3935
3936 if (!(ac->ac_flags & EXT4_MB_HINT_DATA))
3937 return;
3938
Theodore Ts'o4ba74d02009-08-09 22:01:13 -04003939 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))
3940 return;
3941
Theodore Ts'o53accfa2011-09-09 18:48:51 -04003942 size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len);
Theodore Ts'o50797482009-09-18 13:34:02 -04003943 isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1)
3944 >> bsbits;
Alex Tomasc9de5602008-01-29 00:19:52 -05003945
Theodore Ts'o50797482009-09-18 13:34:02 -04003946 if ((size == isize) &&
3947 !ext4_fs_is_busy(sbi) &&
3948 (atomic_read(&ac->ac_inode->i_writecount) == 0)) {
3949 ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC;
3950 return;
3951 }
3952
Robin Dongebbe0272011-10-26 05:14:27 -04003953 if (sbi->s_mb_group_prealloc <= 0) {
3954 ac->ac_flags |= EXT4_MB_STREAM_ALLOC;
3955 return;
3956 }
3957
Alex Tomasc9de5602008-01-29 00:19:52 -05003958 /* don't use group allocation for large files */
Theodore Ts'o71780572009-09-28 00:06:20 -04003959 size = max(size, isize);
Tao Macc483f12010-03-01 19:06:35 -05003960 if (size > sbi->s_mb_stream_request) {
Theodore Ts'o4ba74d02009-08-09 22:01:13 -04003961 ac->ac_flags |= EXT4_MB_STREAM_ALLOC;
Alex Tomasc9de5602008-01-29 00:19:52 -05003962 return;
Theodore Ts'o4ba74d02009-08-09 22:01:13 -04003963 }
Alex Tomasc9de5602008-01-29 00:19:52 -05003964
3965 BUG_ON(ac->ac_lg != NULL);
3966 /*
3967 * locality group prealloc space are per cpu. The reason for having
3968 * per cpu locality group is to reduce the contention between block
3969 * request from multiple CPUs.
3970 */
Christoph Lameterca0c9582009-10-03 19:48:22 +09003971 ac->ac_lg = __this_cpu_ptr(sbi->s_locality_groups);
Alex Tomasc9de5602008-01-29 00:19:52 -05003972
3973 /* we're going to use group allocation */
3974 ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC;
3975
3976 /* serialize all allocations in the group */
3977 mutex_lock(&ac->ac_lg->lg_mutex);
3978}
3979
Eric Sandeen4ddfef72008-04-29 08:11:12 -04003980static noinline_for_stack int
3981ext4_mb_initialize_context(struct ext4_allocation_context *ac,
Alex Tomasc9de5602008-01-29 00:19:52 -05003982 struct ext4_allocation_request *ar)
3983{
3984 struct super_block *sb = ar->inode->i_sb;
3985 struct ext4_sb_info *sbi = EXT4_SB(sb);
3986 struct ext4_super_block *es = sbi->s_es;
3987 ext4_group_t group;
Theodore Ts'o498e5f22008-11-05 00:14:04 -05003988 unsigned int len;
3989 ext4_fsblk_t goal;
Alex Tomasc9de5602008-01-29 00:19:52 -05003990 ext4_grpblk_t block;
3991
3992 /* we can't allocate > group size */
3993 len = ar->len;
3994
3995 /* just a dirty hack to filter too big requests */
Theodore Ts'o7137d7a2011-09-09 18:38:51 -04003996 if (len >= EXT4_CLUSTERS_PER_GROUP(sb) - 10)
3997 len = EXT4_CLUSTERS_PER_GROUP(sb) - 10;
Alex Tomasc9de5602008-01-29 00:19:52 -05003998
3999 /* start searching from the goal */
4000 goal = ar->goal;
4001 if (goal < le32_to_cpu(es->s_first_data_block) ||
4002 goal >= ext4_blocks_count(es))
4003 goal = le32_to_cpu(es->s_first_data_block);
4004 ext4_get_group_no_and_offset(sb, goal, &group, &block);
4005
4006 /* set up allocation goals */
Theodore Ts'o833576b2009-07-13 09:45:52 -04004007 memset(ac, 0, sizeof(struct ext4_allocation_context));
Theodore Ts'o53accfa2011-09-09 18:48:51 -04004008 ac->ac_b_ex.fe_logical = ar->logical & ~(sbi->s_cluster_ratio - 1);
Alex Tomasc9de5602008-01-29 00:19:52 -05004009 ac->ac_status = AC_STATUS_CONTINUE;
Alex Tomasc9de5602008-01-29 00:19:52 -05004010 ac->ac_sb = sb;
4011 ac->ac_inode = ar->inode;
Theodore Ts'o53accfa2011-09-09 18:48:51 -04004012 ac->ac_o_ex.fe_logical = ac->ac_b_ex.fe_logical;
Alex Tomasc9de5602008-01-29 00:19:52 -05004013 ac->ac_o_ex.fe_group = group;
4014 ac->ac_o_ex.fe_start = block;
4015 ac->ac_o_ex.fe_len = len;
Theodore Ts'o53accfa2011-09-09 18:48:51 -04004016 ac->ac_g_ex = ac->ac_o_ex;
Alex Tomasc9de5602008-01-29 00:19:52 -05004017 ac->ac_flags = ar->flags;
Alex Tomasc9de5602008-01-29 00:19:52 -05004018
4019 /* we have to define context: we'll we work with a file or
4020 * locality group. this is a policy, actually */
4021 ext4_mb_group_or_file(ac);
4022
Theodore Ts'o6ba495e2009-09-18 13:38:55 -04004023 mb_debug(1, "init ac: %u blocks @ %u, goal %u, flags %x, 2^%d, "
Alex Tomasc9de5602008-01-29 00:19:52 -05004024 "left: %u/%u, right %u/%u to %swritable\n",
4025 (unsigned) ar->len, (unsigned) ar->logical,
4026 (unsigned) ar->goal, ac->ac_flags, ac->ac_2order,
4027 (unsigned) ar->lleft, (unsigned) ar->pleft,
4028 (unsigned) ar->lright, (unsigned) ar->pright,
4029 atomic_read(&ar->inode->i_writecount) ? "" : "non-");
4030 return 0;
4031
4032}
4033
Aneesh Kumar K.V6be2ded2008-07-23 14:14:05 -04004034static noinline_for_stack void
4035ext4_mb_discard_lg_preallocations(struct super_block *sb,
4036 struct ext4_locality_group *lg,
4037 int order, int total_entries)
4038{
4039 ext4_group_t group = 0;
4040 struct ext4_buddy e4b;
4041 struct list_head discard_list;
4042 struct ext4_prealloc_space *pa, *tmp;
Aneesh Kumar K.V6be2ded2008-07-23 14:14:05 -04004043
Theodore Ts'o6ba495e2009-09-18 13:38:55 -04004044 mb_debug(1, "discard locality group preallocation\n");
Aneesh Kumar K.V6be2ded2008-07-23 14:14:05 -04004045
4046 INIT_LIST_HEAD(&discard_list);
Aneesh Kumar K.V6be2ded2008-07-23 14:14:05 -04004047
4048 spin_lock(&lg->lg_prealloc_lock);
4049 list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[order],
4050 pa_inode_list) {
4051 spin_lock(&pa->pa_lock);
4052 if (atomic_read(&pa->pa_count)) {
4053 /*
4054 * This is the pa that we just used
4055 * for block allocation. So don't
4056 * free that
4057 */
4058 spin_unlock(&pa->pa_lock);
4059 continue;
4060 }
4061 if (pa->pa_deleted) {
4062 spin_unlock(&pa->pa_lock);
4063 continue;
4064 }
4065 /* only lg prealloc space */
Aneesh Kumar K.Vcc0fb9a2009-03-27 17:16:58 -04004066 BUG_ON(pa->pa_type != MB_GROUP_PA);
Aneesh Kumar K.V6be2ded2008-07-23 14:14:05 -04004067
4068 /* seems this one can be freed ... */
4069 pa->pa_deleted = 1;
4070 spin_unlock(&pa->pa_lock);
4071
4072 list_del_rcu(&pa->pa_inode_list);
4073 list_add(&pa->u.pa_tmp_list, &discard_list);
4074
4075 total_entries--;
4076 if (total_entries <= 5) {
4077 /*
4078 * we want to keep only 5 entries
4079 * allowing it to grow to 8. This
4080 * mak sure we don't call discard
4081 * soon for this list.
4082 */
4083 break;
4084 }
4085 }
4086 spin_unlock(&lg->lg_prealloc_lock);
4087
4088 list_for_each_entry_safe(pa, tmp, &discard_list, u.pa_tmp_list) {
4089
4090 ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, NULL);
4091 if (ext4_mb_load_buddy(sb, group, &e4b)) {
Eric Sandeen12062dd2010-02-15 14:19:27 -05004092 ext4_error(sb, "Error loading buddy information for %u",
4093 group);
Aneesh Kumar K.V6be2ded2008-07-23 14:14:05 -04004094 continue;
4095 }
4096 ext4_lock_group(sb, group);
4097 list_del(&pa->pa_group_list);
Eric Sandeen3e1e5f52010-10-27 21:30:07 -04004098 ext4_mb_release_group_pa(&e4b, pa);
Aneesh Kumar K.V6be2ded2008-07-23 14:14:05 -04004099 ext4_unlock_group(sb, group);
4100
Jing Zhange39e07f2010-05-14 00:00:00 -04004101 ext4_mb_unload_buddy(&e4b);
Aneesh Kumar K.V6be2ded2008-07-23 14:14:05 -04004102 list_del(&pa->u.pa_tmp_list);
4103 call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback);
4104 }
Aneesh Kumar K.V6be2ded2008-07-23 14:14:05 -04004105}
4106
4107/*
4108 * We have incremented pa_count. So it cannot be freed at this
4109 * point. Also we hold lg_mutex. So no parallel allocation is
4110 * possible from this lg. That means pa_free cannot be updated.
4111 *
4112 * A parallel ext4_mb_discard_group_preallocations is possible.
4113 * which can cause the lg_prealloc_list to be updated.
4114 */
4115
4116static void ext4_mb_add_n_trim(struct ext4_allocation_context *ac)
4117{
4118 int order, added = 0, lg_prealloc_count = 1;
4119 struct super_block *sb = ac->ac_sb;
4120 struct ext4_locality_group *lg = ac->ac_lg;
4121 struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa;
4122
4123 order = fls(pa->pa_free) - 1;
4124 if (order > PREALLOC_TB_SIZE - 1)
4125 /* The max size of hash table is PREALLOC_TB_SIZE */
4126 order = PREALLOC_TB_SIZE - 1;
4127 /* Add the prealloc space to lg */
4128 rcu_read_lock();
4129 list_for_each_entry_rcu(tmp_pa, &lg->lg_prealloc_list[order],
4130 pa_inode_list) {
4131 spin_lock(&tmp_pa->pa_lock);
4132 if (tmp_pa->pa_deleted) {
Theodore Ts'oe7c9e3e2009-03-27 19:43:21 -04004133 spin_unlock(&tmp_pa->pa_lock);
Aneesh Kumar K.V6be2ded2008-07-23 14:14:05 -04004134 continue;
4135 }
4136 if (!added && pa->pa_free < tmp_pa->pa_free) {
4137 /* Add to the tail of the previous entry */
4138 list_add_tail_rcu(&pa->pa_inode_list,
4139 &tmp_pa->pa_inode_list);
4140 added = 1;
4141 /*
4142 * we want to count the total
4143 * number of entries in the list
4144 */
4145 }
4146 spin_unlock(&tmp_pa->pa_lock);
4147 lg_prealloc_count++;
4148 }
4149 if (!added)
4150 list_add_tail_rcu(&pa->pa_inode_list,
4151 &lg->lg_prealloc_list[order]);
4152 rcu_read_unlock();
4153
4154 /* Now trim the list to be not more than 8 elements */
4155 if (lg_prealloc_count > 8) {
4156 ext4_mb_discard_lg_preallocations(sb, lg,
4157 order, lg_prealloc_count);
4158 return;
4159 }
4160 return ;
4161}
4162
Alex Tomasc9de5602008-01-29 00:19:52 -05004163/*
4164 * release all resource we used in allocation
4165 */
4166static int ext4_mb_release_context(struct ext4_allocation_context *ac)
4167{
Theodore Ts'o53accfa2011-09-09 18:48:51 -04004168 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
Aneesh Kumar K.V6be2ded2008-07-23 14:14:05 -04004169 struct ext4_prealloc_space *pa = ac->ac_pa;
4170 if (pa) {
Aneesh Kumar K.Vcc0fb9a2009-03-27 17:16:58 -04004171 if (pa->pa_type == MB_GROUP_PA) {
Alex Tomasc9de5602008-01-29 00:19:52 -05004172 /* see comment in ext4_mb_use_group_pa() */
Aneesh Kumar K.V6be2ded2008-07-23 14:14:05 -04004173 spin_lock(&pa->pa_lock);
Theodore Ts'o53accfa2011-09-09 18:48:51 -04004174 pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len);
4175 pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len);
Aneesh Kumar K.V6be2ded2008-07-23 14:14:05 -04004176 pa->pa_free -= ac->ac_b_ex.fe_len;
4177 pa->pa_len -= ac->ac_b_ex.fe_len;
4178 spin_unlock(&pa->pa_lock);
Alex Tomasc9de5602008-01-29 00:19:52 -05004179 }
Alex Tomasc9de5602008-01-29 00:19:52 -05004180 }
Aneesh Kumar K.Vba443912009-02-10 11:14:34 -05004181 if (pa) {
4182 /*
4183 * We want to add the pa to the right bucket.
4184 * Remove it from the list and while adding
4185 * make sure the list to which we are adding
Amir Goldstein44183d42011-05-09 21:52:36 -04004186 * doesn't grow big.
Aneesh Kumar K.Vba443912009-02-10 11:14:34 -05004187 */
Aneesh Kumar K.Vcc0fb9a2009-03-27 17:16:58 -04004188 if ((pa->pa_type == MB_GROUP_PA) && likely(pa->pa_free)) {
Aneesh Kumar K.Vba443912009-02-10 11:14:34 -05004189 spin_lock(pa->pa_obj_lock);
4190 list_del_rcu(&pa->pa_inode_list);
4191 spin_unlock(pa->pa_obj_lock);
4192 ext4_mb_add_n_trim(ac);
4193 }
4194 ext4_mb_put_pa(ac, ac->ac_sb, pa);
4195 }
Alex Tomasc9de5602008-01-29 00:19:52 -05004196 if (ac->ac_bitmap_page)
4197 page_cache_release(ac->ac_bitmap_page);
4198 if (ac->ac_buddy_page)
4199 page_cache_release(ac->ac_buddy_page);
4200 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)
4201 mutex_unlock(&ac->ac_lg->lg_mutex);
4202 ext4_mb_collect_stats(ac);
4203 return 0;
4204}
4205
4206static int ext4_mb_discard_preallocations(struct super_block *sb, int needed)
4207{
Theodore Ts'o8df96752009-05-01 08:50:38 -04004208 ext4_group_t i, ngroups = ext4_get_groups_count(sb);
Alex Tomasc9de5602008-01-29 00:19:52 -05004209 int ret;
4210 int freed = 0;
4211
Theodore Ts'o9bffad12009-06-17 11:48:11 -04004212 trace_ext4_mb_discard_preallocations(sb, needed);
Theodore Ts'o8df96752009-05-01 08:50:38 -04004213 for (i = 0; i < ngroups && needed > 0; i++) {
Alex Tomasc9de5602008-01-29 00:19:52 -05004214 ret = ext4_mb_discard_group_preallocations(sb, i, needed);
4215 freed += ret;
4216 needed -= ret;
4217 }
4218
4219 return freed;
4220}
4221
4222/*
4223 * Main entry point into mballoc to allocate blocks
4224 * it tries to use preallocation first, then falls back
4225 * to usual allocation
4226 */
4227ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle,
Aditya Kali6c7a1202010-08-05 16:22:24 -04004228 struct ext4_allocation_request *ar, int *errp)
Alex Tomasc9de5602008-01-29 00:19:52 -05004229{
Aneesh Kumar K.V6bc6e632008-10-10 09:39:00 -04004230 int freed;
Eric Sandeen256bdb42008-02-10 01:13:33 -05004231 struct ext4_allocation_context *ac = NULL;
Alex Tomasc9de5602008-01-29 00:19:52 -05004232 struct ext4_sb_info *sbi;
4233 struct super_block *sb;
4234 ext4_fsblk_t block = 0;
Mingming Cao60e58e02009-01-22 18:13:05 +01004235 unsigned int inquota = 0;
Theodore Ts'o53accfa2011-09-09 18:48:51 -04004236 unsigned int reserv_clstrs = 0;
Alex Tomasc9de5602008-01-29 00:19:52 -05004237
4238 sb = ar->inode->i_sb;
4239 sbi = EXT4_SB(sb);
4240
Theodore Ts'o9bffad12009-06-17 11:48:11 -04004241 trace_ext4_request_blocks(ar);
Theodore Ts'oba80b102009-01-03 20:03:21 -05004242
Dmitry Monakhov45dc63e2011-10-20 20:07:23 -04004243 /* Allow to use superuser reservation for quota file */
4244 if (IS_NOQUOTA(ar->inode))
4245 ar->flags |= EXT4_MB_USE_ROOT_BLOCKS;
4246
Mingming Cao60e58e02009-01-22 18:13:05 +01004247 /*
4248 * For delayed allocation, we could skip the ENOSPC and
4249 * EDQUOT check, as blocks and quotas have been already
4250 * reserved when data being copied into pagecache.
4251 */
Theodore Ts'of2321092011-01-10 12:12:36 -05004252 if (ext4_test_inode_state(ar->inode, EXT4_STATE_DELALLOC_RESERVED))
Mingming Cao60e58e02009-01-22 18:13:05 +01004253 ar->flags |= EXT4_MB_DELALLOC_RESERVED;
4254 else {
4255 /* Without delayed allocation we need to verify
4256 * there is enough free blocks to do block allocation
4257 * and verify allocation doesn't exceed the quota limits.
Mingming Caod2a17632008-07-14 17:52:37 -04004258 */
Allison Henderson55f020d2011-05-25 07:41:26 -04004259 while (ar->len &&
Theodore Ts'oe7d5f312011-09-09 19:14:51 -04004260 ext4_claim_free_clusters(sbi, ar->len, ar->flags)) {
Allison Henderson55f020d2011-05-25 07:41:26 -04004261
Aneesh Kumar K.V030ba6b2008-09-08 23:14:50 -04004262 /* let others to free the space */
4263 yield();
4264 ar->len = ar->len >> 1;
4265 }
4266 if (!ar->len) {
Aneesh Kumar K.Va30d542a2008-10-09 10:56:23 -04004267 *errp = -ENOSPC;
4268 return 0;
4269 }
Theodore Ts'o53accfa2011-09-09 18:48:51 -04004270 reserv_clstrs = ar->len;
Allison Henderson55f020d2011-05-25 07:41:26 -04004271 if (ar->flags & EXT4_MB_USE_ROOT_BLOCKS) {
Theodore Ts'o53accfa2011-09-09 18:48:51 -04004272 dquot_alloc_block_nofail(ar->inode,
4273 EXT4_C2B(sbi, ar->len));
Allison Henderson55f020d2011-05-25 07:41:26 -04004274 } else {
4275 while (ar->len &&
Theodore Ts'o53accfa2011-09-09 18:48:51 -04004276 dquot_alloc_block(ar->inode,
4277 EXT4_C2B(sbi, ar->len))) {
Allison Henderson55f020d2011-05-25 07:41:26 -04004278
4279 ar->flags |= EXT4_MB_HINT_NOPREALLOC;
4280 ar->len--;
4281 }
Mingming Cao60e58e02009-01-22 18:13:05 +01004282 }
4283 inquota = ar->len;
4284 if (ar->len == 0) {
4285 *errp = -EDQUOT;
Aditya Kali6c7a1202010-08-05 16:22:24 -04004286 goto out;
Mingming Cao60e58e02009-01-22 18:13:05 +01004287 }
Mingming Caod2a17632008-07-14 17:52:37 -04004288 }
Mingming Caod2a17632008-07-14 17:52:37 -04004289
Eric Sandeen256bdb42008-02-10 01:13:33 -05004290 ac = kmem_cache_alloc(ext4_ac_cachep, GFP_NOFS);
Theodore Ts'o833576b2009-07-13 09:45:52 -04004291 if (!ac) {
Shen Feng363d4252008-07-11 19:27:31 -04004292 ar->len = 0;
Eric Sandeen256bdb42008-02-10 01:13:33 -05004293 *errp = -ENOMEM;
Aditya Kali6c7a1202010-08-05 16:22:24 -04004294 goto out;
Eric Sandeen256bdb42008-02-10 01:13:33 -05004295 }
4296
Eric Sandeen256bdb42008-02-10 01:13:33 -05004297 *errp = ext4_mb_initialize_context(ac, ar);
Alex Tomasc9de5602008-01-29 00:19:52 -05004298 if (*errp) {
4299 ar->len = 0;
Aditya Kali6c7a1202010-08-05 16:22:24 -04004300 goto out;
Alex Tomasc9de5602008-01-29 00:19:52 -05004301 }
4302
Eric Sandeen256bdb42008-02-10 01:13:33 -05004303 ac->ac_op = EXT4_MB_HISTORY_PREALLOC;
4304 if (!ext4_mb_use_preallocated(ac)) {
Eric Sandeen256bdb42008-02-10 01:13:33 -05004305 ac->ac_op = EXT4_MB_HISTORY_ALLOC;
4306 ext4_mb_normalize_request(ac, ar);
Alex Tomasc9de5602008-01-29 00:19:52 -05004307repeat:
4308 /* allocate space in core */
Aditya Kali6c7a1202010-08-05 16:22:24 -04004309 *errp = ext4_mb_regular_allocator(ac);
4310 if (*errp)
4311 goto errout;
Alex Tomasc9de5602008-01-29 00:19:52 -05004312
4313 /* as we've just preallocated more space than
4314 * user requested orinally, we store allocated
4315 * space in a special descriptor */
Eric Sandeen256bdb42008-02-10 01:13:33 -05004316 if (ac->ac_status == AC_STATUS_FOUND &&
4317 ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len)
4318 ext4_mb_new_preallocation(ac);
Alex Tomasc9de5602008-01-29 00:19:52 -05004319 }
Eric Sandeen256bdb42008-02-10 01:13:33 -05004320 if (likely(ac->ac_status == AC_STATUS_FOUND)) {
Theodore Ts'o53accfa2011-09-09 18:48:51 -04004321 *errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_clstrs);
Aditya Kali6c7a1202010-08-05 16:22:24 -04004322 if (*errp == -EAGAIN) {
Aneesh Kumar K.V8556e8f2009-01-05 21:46:55 -05004323 /*
4324 * drop the reference that we took
4325 * in ext4_mb_use_best_found
4326 */
4327 ext4_mb_release_context(ac);
Aneesh Kumar K.V519deca2008-05-15 14:43:20 -04004328 ac->ac_b_ex.fe_group = 0;
4329 ac->ac_b_ex.fe_start = 0;
4330 ac->ac_b_ex.fe_len = 0;
4331 ac->ac_status = AC_STATUS_CONTINUE;
4332 goto repeat;
Aditya Kali6c7a1202010-08-05 16:22:24 -04004333 } else if (*errp)
4334 errout:
Curt Wohlgemuthb8441672009-12-08 22:18:25 -05004335 ext4_discard_allocated_blocks(ac);
Aditya Kali6c7a1202010-08-05 16:22:24 -04004336 else {
Aneesh Kumar K.V519deca2008-05-15 14:43:20 -04004337 block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);
4338 ar->len = ac->ac_b_ex.fe_len;
4339 }
Alex Tomasc9de5602008-01-29 00:19:52 -05004340 } else {
Eric Sandeen256bdb42008-02-10 01:13:33 -05004341 freed = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len);
Alex Tomasc9de5602008-01-29 00:19:52 -05004342 if (freed)
4343 goto repeat;
4344 *errp = -ENOSPC;
Aditya Kali6c7a1202010-08-05 16:22:24 -04004345 }
4346
4347 if (*errp) {
Eric Sandeen256bdb42008-02-10 01:13:33 -05004348 ac->ac_b_ex.fe_len = 0;
Alex Tomasc9de5602008-01-29 00:19:52 -05004349 ar->len = 0;
Eric Sandeen256bdb42008-02-10 01:13:33 -05004350 ext4_mb_show_ac(ac);
Alex Tomasc9de5602008-01-29 00:19:52 -05004351 }
Eric Sandeen256bdb42008-02-10 01:13:33 -05004352 ext4_mb_release_context(ac);
Aditya Kali6c7a1202010-08-05 16:22:24 -04004353out:
4354 if (ac)
4355 kmem_cache_free(ext4_ac_cachep, ac);
Mingming Cao60e58e02009-01-22 18:13:05 +01004356 if (inquota && ar->len < inquota)
Theodore Ts'o53accfa2011-09-09 18:48:51 -04004357 dquot_free_block(ar->inode, EXT4_C2B(sbi, inquota - ar->len));
Aneesh Kumar K.V0087d9f2009-01-05 21:49:12 -05004358 if (!ar->len) {
Theodore Ts'of2321092011-01-10 12:12:36 -05004359 if (!ext4_test_inode_state(ar->inode,
4360 EXT4_STATE_DELALLOC_RESERVED))
Aneesh Kumar K.V0087d9f2009-01-05 21:49:12 -05004361 /* release all the reserved blocks if non delalloc */
Theodore Ts'o57042652011-09-09 18:56:51 -04004362 percpu_counter_sub(&sbi->s_dirtyclusters_counter,
Theodore Ts'o53accfa2011-09-09 18:48:51 -04004363 reserv_clstrs);
Aneesh Kumar K.V0087d9f2009-01-05 21:49:12 -05004364 }
Alex Tomasc9de5602008-01-29 00:19:52 -05004365
Theodore Ts'o9bffad12009-06-17 11:48:11 -04004366 trace_ext4_allocate_blocks(ar, (unsigned long long)block);
Theodore Ts'oba80b102009-01-03 20:03:21 -05004367
Alex Tomasc9de5602008-01-29 00:19:52 -05004368 return block;
4369}
Alex Tomasc9de5602008-01-29 00:19:52 -05004370
Aneesh Kumar K.Vc8940582008-10-16 10:14:27 -04004371/*
4372 * We can merge two free data extents only if the physical blocks
4373 * are contiguous, AND the extents were freed by the same transaction,
4374 * AND the blocks are associated with the same group.
4375 */
4376static int can_merge(struct ext4_free_data *entry1,
4377 struct ext4_free_data *entry2)
4378{
Bobi Jam18aadd42012-02-20 17:53:02 -05004379 if ((entry1->efd_tid == entry2->efd_tid) &&
4380 (entry1->efd_group == entry2->efd_group) &&
4381 ((entry1->efd_start_cluster + entry1->efd_count) == entry2->efd_start_cluster))
Aneesh Kumar K.Vc8940582008-10-16 10:14:27 -04004382 return 1;
4383 return 0;
4384}
4385
Eric Sandeen4ddfef72008-04-29 08:11:12 -04004386static noinline_for_stack int
4387ext4_mb_free_metadata(handle_t *handle, struct ext4_buddy *e4b,
Aneesh Kumar K.V7a2fcbf2009-01-05 21:36:55 -05004388 struct ext4_free_data *new_entry)
Alex Tomasc9de5602008-01-29 00:19:52 -05004389{
Theodore Ts'oe29136f2010-06-29 12:54:28 -04004390 ext4_group_t group = e4b->bd_group;
Theodore Ts'o84130192011-09-09 18:50:51 -04004391 ext4_grpblk_t cluster;
Aneesh Kumar K.V7a2fcbf2009-01-05 21:36:55 -05004392 struct ext4_free_data *entry;
Alex Tomasc9de5602008-01-29 00:19:52 -05004393 struct ext4_group_info *db = e4b->bd_info;
4394 struct super_block *sb = e4b->bd_sb;
4395 struct ext4_sb_info *sbi = EXT4_SB(sb);
Aneesh Kumar K.Vc8940582008-10-16 10:14:27 -04004396 struct rb_node **n = &db->bb_free_root.rb_node, *node;
4397 struct rb_node *parent = NULL, *new_node;
4398
Frank Mayhar03901312009-01-07 00:06:22 -05004399 BUG_ON(!ext4_handle_valid(handle));
Alex Tomasc9de5602008-01-29 00:19:52 -05004400 BUG_ON(e4b->bd_bitmap_page == NULL);
4401 BUG_ON(e4b->bd_buddy_page == NULL);
4402
Bobi Jam18aadd42012-02-20 17:53:02 -05004403 new_node = &new_entry->efd_node;
4404 cluster = new_entry->efd_start_cluster;
Aneesh Kumar K.Vc8940582008-10-16 10:14:27 -04004405
Aneesh Kumar K.Vc8940582008-10-16 10:14:27 -04004406 if (!*n) {
4407 /* first free block exent. We need to
4408 protect buddy cache from being freed,
4409 * otherwise we'll refresh it from
4410 * on-disk bitmap and lose not-yet-available
4411 * blocks */
4412 page_cache_get(e4b->bd_buddy_page);
4413 page_cache_get(e4b->bd_bitmap_page);
4414 }
4415 while (*n) {
4416 parent = *n;
Bobi Jam18aadd42012-02-20 17:53:02 -05004417 entry = rb_entry(parent, struct ext4_free_data, efd_node);
4418 if (cluster < entry->efd_start_cluster)
Aneesh Kumar K.Vc8940582008-10-16 10:14:27 -04004419 n = &(*n)->rb_left;
Bobi Jam18aadd42012-02-20 17:53:02 -05004420 else if (cluster >= (entry->efd_start_cluster + entry->efd_count))
Aneesh Kumar K.Vc8940582008-10-16 10:14:27 -04004421 n = &(*n)->rb_right;
4422 else {
Theodore Ts'oe29136f2010-06-29 12:54:28 -04004423 ext4_grp_locked_error(sb, group, 0,
Theodore Ts'o84130192011-09-09 18:50:51 -04004424 ext4_group_first_block_no(sb, group) +
4425 EXT4_C2B(sbi, cluster),
Theodore Ts'oe29136f2010-06-29 12:54:28 -04004426 "Block already on to-be-freed list");
Aneesh Kumar K.Vc8940582008-10-16 10:14:27 -04004427 return 0;
Alex Tomasc9de5602008-01-29 00:19:52 -05004428 }
4429 }
Aneesh Kumar K.Vc8940582008-10-16 10:14:27 -04004430
4431 rb_link_node(new_node, parent, n);
4432 rb_insert_color(new_node, &db->bb_free_root);
4433
4434 /* Now try to see the extent can be merged to left and right */
4435 node = rb_prev(new_node);
4436 if (node) {
Bobi Jam18aadd42012-02-20 17:53:02 -05004437 entry = rb_entry(node, struct ext4_free_data, efd_node);
Aneesh Kumar K.Vc8940582008-10-16 10:14:27 -04004438 if (can_merge(entry, new_entry)) {
Bobi Jam18aadd42012-02-20 17:53:02 -05004439 new_entry->efd_start_cluster = entry->efd_start_cluster;
4440 new_entry->efd_count += entry->efd_count;
Aneesh Kumar K.Vc8940582008-10-16 10:14:27 -04004441 rb_erase(node, &(db->bb_free_root));
Bobi Jam18aadd42012-02-20 17:53:02 -05004442 ext4_journal_callback_del(handle, &entry->efd_jce);
4443 kmem_cache_free(ext4_free_data_cachep, entry);
Aneesh Kumar K.Vc8940582008-10-16 10:14:27 -04004444 }
4445 }
4446
4447 node = rb_next(new_node);
4448 if (node) {
Bobi Jam18aadd42012-02-20 17:53:02 -05004449 entry = rb_entry(node, struct ext4_free_data, efd_node);
Aneesh Kumar K.Vc8940582008-10-16 10:14:27 -04004450 if (can_merge(new_entry, entry)) {
Bobi Jam18aadd42012-02-20 17:53:02 -05004451 new_entry->efd_count += entry->efd_count;
Aneesh Kumar K.Vc8940582008-10-16 10:14:27 -04004452 rb_erase(node, &(db->bb_free_root));
Bobi Jam18aadd42012-02-20 17:53:02 -05004453 ext4_journal_callback_del(handle, &entry->efd_jce);
4454 kmem_cache_free(ext4_free_data_cachep, entry);
Aneesh Kumar K.Vc8940582008-10-16 10:14:27 -04004455 }
4456 }
Theodore Ts'o3e624fc2008-10-16 20:00:24 -04004457 /* Add the extent to transaction's private list */
Bobi Jam18aadd42012-02-20 17:53:02 -05004458 ext4_journal_callback_add(handle, ext4_free_data_callback,
4459 &new_entry->efd_jce);
Alex Tomasc9de5602008-01-29 00:19:52 -05004460 return 0;
4461}
4462
Theodore Ts'o44338712009-11-22 07:44:56 -05004463/**
4464 * ext4_free_blocks() -- Free given blocks and update quota
4465 * @handle: handle for this transaction
4466 * @inode: inode
4467 * @block: start physical block to free
4468 * @count: number of blocks to count
Yongqiang Yang5def1362011-06-05 23:26:40 -04004469 * @flags: flags used by ext4_free_blocks
Alex Tomasc9de5602008-01-29 00:19:52 -05004470 */
Theodore Ts'o44338712009-11-22 07:44:56 -05004471void ext4_free_blocks(handle_t *handle, struct inode *inode,
Theodore Ts'oe6362602009-11-23 07:17:05 -05004472 struct buffer_head *bh, ext4_fsblk_t block,
4473 unsigned long count, int flags)
Alex Tomasc9de5602008-01-29 00:19:52 -05004474{
Aneesh Kumar K.V26346ff2008-02-10 01:10:04 -05004475 struct buffer_head *bitmap_bh = NULL;
Alex Tomasc9de5602008-01-29 00:19:52 -05004476 struct super_block *sb = inode->i_sb;
Alex Tomasc9de5602008-01-29 00:19:52 -05004477 struct ext4_group_desc *gdp;
Theodore Ts'o44338712009-11-22 07:44:56 -05004478 unsigned long freed = 0;
Theodore Ts'o498e5f22008-11-05 00:14:04 -05004479 unsigned int overflow;
Alex Tomasc9de5602008-01-29 00:19:52 -05004480 ext4_grpblk_t bit;
4481 struct buffer_head *gd_bh;
4482 ext4_group_t block_group;
4483 struct ext4_sb_info *sbi;
4484 struct ext4_buddy e4b;
Theodore Ts'o84130192011-09-09 18:50:51 -04004485 unsigned int count_clusters;
Alex Tomasc9de5602008-01-29 00:19:52 -05004486 int err = 0;
4487 int ret;
4488
Theodore Ts'oe6362602009-11-23 07:17:05 -05004489 if (bh) {
4490 if (block)
4491 BUG_ON(block != bh->b_blocknr);
4492 else
4493 block = bh->b_blocknr;
4494 }
Alex Tomasc9de5602008-01-29 00:19:52 -05004495
Alex Tomasc9de5602008-01-29 00:19:52 -05004496 sbi = EXT4_SB(sb);
Theodore Ts'o1f2acb62010-01-22 17:40:42 -05004497 if (!(flags & EXT4_FREE_BLOCKS_VALIDATED) &&
4498 !ext4_data_block_valid(sbi, block, count)) {
Eric Sandeen12062dd2010-02-15 14:19:27 -05004499 ext4_error(sb, "Freeing blocks not in datazone - "
Theodore Ts'o1f2acb62010-01-22 17:40:42 -05004500 "block = %llu, count = %lu", block, count);
Alex Tomasc9de5602008-01-29 00:19:52 -05004501 goto error_return;
4502 }
4503
Theodore Ts'o0610b6e2009-06-15 03:45:05 -04004504 ext4_debug("freeing block %llu\n", block);
Theodore Ts'oe6362602009-11-23 07:17:05 -05004505 trace_ext4_free_blocks(inode, block, count, flags);
4506
4507 if (flags & EXT4_FREE_BLOCKS_FORGET) {
4508 struct buffer_head *tbh = bh;
4509 int i;
4510
4511 BUG_ON(bh && (count > 1));
4512
4513 for (i = 0; i < count; i++) {
4514 if (!bh)
4515 tbh = sb_find_get_block(inode->i_sb,
4516 block + i);
Namhyung Kim87783692010-10-27 21:30:11 -04004517 if (unlikely(!tbh))
4518 continue;
Theodore Ts'o60e66792010-05-17 07:00:00 -04004519 ext4_forget(handle, flags & EXT4_FREE_BLOCKS_METADATA,
Theodore Ts'oe6362602009-11-23 07:17:05 -05004520 inode, tbh, block + i);
4521 }
4522 }
4523
Theodore Ts'o60e66792010-05-17 07:00:00 -04004524 /*
Theodore Ts'oe6362602009-11-23 07:17:05 -05004525 * We need to make sure we don't reuse the freed block until
4526 * after the transaction is committed, which we can do by
4527 * treating the block as metadata, below. We make an
4528 * exception if the inode is to be written in writeback mode
4529 * since writeback mode has weak data consistency guarantees.
4530 */
4531 if (!ext4_should_writeback_data(inode))
4532 flags |= EXT4_FREE_BLOCKS_METADATA;
Alex Tomasc9de5602008-01-29 00:19:52 -05004533
Theodore Ts'o84130192011-09-09 18:50:51 -04004534 /*
4535 * If the extent to be freed does not begin on a cluster
4536 * boundary, we need to deal with partial clusters at the
4537 * beginning and end of the extent. Normally we will free
4538 * blocks at the beginning or the end unless we are explicitly
4539 * requested to avoid doing so.
4540 */
4541 overflow = block & (sbi->s_cluster_ratio - 1);
4542 if (overflow) {
4543 if (flags & EXT4_FREE_BLOCKS_NOFREE_FIRST_CLUSTER) {
4544 overflow = sbi->s_cluster_ratio - overflow;
4545 block += overflow;
4546 if (count > overflow)
4547 count -= overflow;
4548 else
4549 return;
4550 } else {
4551 block -= overflow;
4552 count += overflow;
4553 }
4554 }
4555 overflow = count & (sbi->s_cluster_ratio - 1);
4556 if (overflow) {
4557 if (flags & EXT4_FREE_BLOCKS_NOFREE_LAST_CLUSTER) {
4558 if (count > overflow)
4559 count -= overflow;
4560 else
4561 return;
4562 } else
4563 count += sbi->s_cluster_ratio - overflow;
4564 }
4565
Alex Tomasc9de5602008-01-29 00:19:52 -05004566do_more:
4567 overflow = 0;
4568 ext4_get_group_no_and_offset(sb, block, &block_group, &bit);
4569
4570 /*
4571 * Check to see if we are freeing blocks across a group
4572 * boundary.
4573 */
Theodore Ts'o84130192011-09-09 18:50:51 -04004574 if (EXT4_C2B(sbi, bit) + count > EXT4_BLOCKS_PER_GROUP(sb)) {
4575 overflow = EXT4_C2B(sbi, bit) + count -
4576 EXT4_BLOCKS_PER_GROUP(sb);
Alex Tomasc9de5602008-01-29 00:19:52 -05004577 count -= overflow;
4578 }
Theodore Ts'o84130192011-09-09 18:50:51 -04004579 count_clusters = EXT4_B2C(sbi, count);
Theodore Ts'o574ca172008-07-11 19:27:31 -04004580 bitmap_bh = ext4_read_block_bitmap(sb, block_group);
Aneesh Kumar K.Vce89f462008-07-23 14:09:29 -04004581 if (!bitmap_bh) {
4582 err = -EIO;
Alex Tomasc9de5602008-01-29 00:19:52 -05004583 goto error_return;
Aneesh Kumar K.Vce89f462008-07-23 14:09:29 -04004584 }
Alex Tomasc9de5602008-01-29 00:19:52 -05004585 gdp = ext4_get_group_desc(sb, block_group, &gd_bh);
Aneesh Kumar K.Vce89f462008-07-23 14:09:29 -04004586 if (!gdp) {
4587 err = -EIO;
Alex Tomasc9de5602008-01-29 00:19:52 -05004588 goto error_return;
Aneesh Kumar K.Vce89f462008-07-23 14:09:29 -04004589 }
Alex Tomasc9de5602008-01-29 00:19:52 -05004590
4591 if (in_range(ext4_block_bitmap(sb, gdp), block, count) ||
4592 in_range(ext4_inode_bitmap(sb, gdp), block, count) ||
4593 in_range(block, ext4_inode_table(sb, gdp),
Theodore Ts'o84130192011-09-09 18:50:51 -04004594 EXT4_SB(sb)->s_itb_per_group) ||
Alex Tomasc9de5602008-01-29 00:19:52 -05004595 in_range(block + count - 1, ext4_inode_table(sb, gdp),
Theodore Ts'o84130192011-09-09 18:50:51 -04004596 EXT4_SB(sb)->s_itb_per_group)) {
Alex Tomasc9de5602008-01-29 00:19:52 -05004597
Eric Sandeen12062dd2010-02-15 14:19:27 -05004598 ext4_error(sb, "Freeing blocks in system zone - "
Theodore Ts'o0610b6e2009-06-15 03:45:05 -04004599 "Block = %llu, count = %lu", block, count);
Aneesh Kumar K.V519deca2008-05-15 14:43:20 -04004600 /* err = 0. ext4_std_error should be a no op */
4601 goto error_return;
Alex Tomasc9de5602008-01-29 00:19:52 -05004602 }
4603
4604 BUFFER_TRACE(bitmap_bh, "getting write access");
4605 err = ext4_journal_get_write_access(handle, bitmap_bh);
4606 if (err)
4607 goto error_return;
4608
4609 /*
4610 * We are about to modify some metadata. Call the journal APIs
4611 * to unshare ->b_data if a currently-committing transaction is
4612 * using it
4613 */
4614 BUFFER_TRACE(gd_bh, "get_write_access");
4615 err = ext4_journal_get_write_access(handle, gd_bh);
4616 if (err)
4617 goto error_return;
Alex Tomasc9de5602008-01-29 00:19:52 -05004618#ifdef AGGRESSIVE_CHECK
4619 {
4620 int i;
Theodore Ts'o84130192011-09-09 18:50:51 -04004621 for (i = 0; i < count_clusters; i++)
Alex Tomasc9de5602008-01-29 00:19:52 -05004622 BUG_ON(!mb_test_bit(bit + i, bitmap_bh->b_data));
4623 }
4624#endif
Theodore Ts'o84130192011-09-09 18:50:51 -04004625 trace_ext4_mballoc_free(sb, inode, block_group, bit, count_clusters);
Alex Tomasc9de5602008-01-29 00:19:52 -05004626
Aneesh Kumar K.V920313a2009-01-05 21:36:19 -05004627 err = ext4_mb_load_buddy(sb, block_group, &e4b);
4628 if (err)
4629 goto error_return;
Theodore Ts'oe6362602009-11-23 07:17:05 -05004630
4631 if ((flags & EXT4_FREE_BLOCKS_METADATA) && ext4_handle_valid(handle)) {
Aneesh Kumar K.V7a2fcbf2009-01-05 21:36:55 -05004632 struct ext4_free_data *new_entry;
4633 /*
4634 * blocks being freed are metadata. these blocks shouldn't
4635 * be used until this transaction is committed
4636 */
Bobi Jam18aadd42012-02-20 17:53:02 -05004637 new_entry = kmem_cache_alloc(ext4_free_data_cachep, GFP_NOFS);
Theodore Ts'ob72143a2010-12-20 07:26:59 -05004638 if (!new_entry) {
4639 err = -ENOMEM;
4640 goto error_return;
4641 }
Bobi Jam18aadd42012-02-20 17:53:02 -05004642 new_entry->efd_start_cluster = bit;
4643 new_entry->efd_group = block_group;
4644 new_entry->efd_count = count_clusters;
4645 new_entry->efd_tid = handle->h_transaction->t_tid;
Aneesh Kumar K.V955ce5f2009-05-02 20:35:09 -04004646
Aneesh Kumar K.V7a2fcbf2009-01-05 21:36:55 -05004647 ext4_lock_group(sb, block_group);
Theodore Ts'o84130192011-09-09 18:50:51 -04004648 mb_clear_bits(bitmap_bh->b_data, bit, count_clusters);
Aneesh Kumar K.V7a2fcbf2009-01-05 21:36:55 -05004649 ext4_mb_free_metadata(handle, &e4b, new_entry);
Alex Tomasc9de5602008-01-29 00:19:52 -05004650 } else {
Aneesh Kumar K.V7a2fcbf2009-01-05 21:36:55 -05004651 /* need to update group_info->bb_free and bitmap
4652 * with group lock held. generate_buddy look at
4653 * them with group lock_held
4654 */
Aneesh Kumar K.V955ce5f2009-05-02 20:35:09 -04004655 ext4_lock_group(sb, block_group);
Theodore Ts'o84130192011-09-09 18:50:51 -04004656 mb_clear_bits(bitmap_bh->b_data, bit, count_clusters);
4657 mb_free_blocks(inode, &e4b, bit, count_clusters);
Alex Tomasc9de5602008-01-29 00:19:52 -05004658 }
4659
Theodore Ts'o021b65b2011-09-09 19:08:51 -04004660 ret = ext4_free_group_clusters(sb, gdp) + count_clusters;
4661 ext4_free_group_clusters_set(sb, gdp, ret);
Alex Tomasc9de5602008-01-29 00:19:52 -05004662 gdp->bg_checksum = ext4_group_desc_csum(sbi, block_group, gdp);
Aneesh Kumar K.V955ce5f2009-05-02 20:35:09 -04004663 ext4_unlock_group(sb, block_group);
Theodore Ts'o57042652011-09-09 18:56:51 -04004664 percpu_counter_add(&sbi->s_freeclusters_counter, count_clusters);
Alex Tomasc9de5602008-01-29 00:19:52 -05004665
Jose R. Santos772cb7c2008-07-11 19:27:31 -04004666 if (sbi->s_log_groups_per_flex) {
4667 ext4_group_t flex_group = ext4_flex_group(sbi, block_group);
Theodore Ts'o24aaa8e2011-09-09 18:58:51 -04004668 atomic_add(count_clusters,
4669 &sbi->s_flex_groups[flex_group].free_clusters);
Jose R. Santos772cb7c2008-07-11 19:27:31 -04004670 }
4671
Jing Zhange39e07f2010-05-14 00:00:00 -04004672 ext4_mb_unload_buddy(&e4b);
Alex Tomasc9de5602008-01-29 00:19:52 -05004673
Theodore Ts'o44338712009-11-22 07:44:56 -05004674 freed += count;
Alex Tomasc9de5602008-01-29 00:19:52 -05004675
Aditya Kali7b415bf2011-09-09 19:04:51 -04004676 if (!(flags & EXT4_FREE_BLOCKS_NO_QUOT_UPDATE))
4677 dquot_free_block(inode, EXT4_C2B(sbi, count_clusters));
4678
Aneesh Kumar K.V7a2fcbf2009-01-05 21:36:55 -05004679 /* We dirtied the bitmap block */
4680 BUFFER_TRACE(bitmap_bh, "dirtied bitmap block");
4681 err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh);
4682
Alex Tomasc9de5602008-01-29 00:19:52 -05004683 /* And the group descriptor block */
4684 BUFFER_TRACE(gd_bh, "dirtied group descriptor block");
Frank Mayhar03901312009-01-07 00:06:22 -05004685 ret = ext4_handle_dirty_metadata(handle, NULL, gd_bh);
Alex Tomasc9de5602008-01-29 00:19:52 -05004686 if (!err)
4687 err = ret;
4688
4689 if (overflow && !err) {
4690 block += count;
4691 count = overflow;
4692 put_bh(bitmap_bh);
4693 goto do_more;
4694 }
Theodore Ts'oa0375152010-06-11 23:14:04 -04004695 ext4_mark_super_dirty(sb);
Alex Tomasc9de5602008-01-29 00:19:52 -05004696error_return:
4697 brelse(bitmap_bh);
4698 ext4_std_error(sb, err);
Alex Tomasc9de5602008-01-29 00:19:52 -05004699 return;
4700}
Lukas Czerner7360d172010-10-27 21:30:12 -04004701
4702/**
Yongqiang Yang05291552011-07-26 21:43:56 -04004703 * ext4_group_add_blocks() -- Add given blocks to an existing group
Amir Goldstein2846e822011-05-09 10:46:41 -04004704 * @handle: handle to this transaction
4705 * @sb: super block
4706 * @block: start physcial block to add to the block group
4707 * @count: number of blocks to free
4708 *
Amir Goldsteine73a3472011-05-09 21:40:01 -04004709 * This marks the blocks as free in the bitmap and buddy.
Amir Goldstein2846e822011-05-09 10:46:41 -04004710 */
Yongqiang Yangcc7365d2011-07-26 21:46:07 -04004711int ext4_group_add_blocks(handle_t *handle, struct super_block *sb,
Amir Goldstein2846e822011-05-09 10:46:41 -04004712 ext4_fsblk_t block, unsigned long count)
4713{
4714 struct buffer_head *bitmap_bh = NULL;
4715 struct buffer_head *gd_bh;
4716 ext4_group_t block_group;
4717 ext4_grpblk_t bit;
4718 unsigned int i;
4719 struct ext4_group_desc *desc;
4720 struct ext4_sb_info *sbi = EXT4_SB(sb);
Amir Goldsteine73a3472011-05-09 21:40:01 -04004721 struct ext4_buddy e4b;
Amir Goldstein2846e822011-05-09 10:46:41 -04004722 int err = 0, ret, blk_free_count;
4723 ext4_grpblk_t blocks_freed;
Amir Goldstein2846e822011-05-09 10:46:41 -04004724
4725 ext4_debug("Adding block(s) %llu-%llu\n", block, block + count - 1);
4726
Yongqiang Yang4740b832011-07-26 21:51:08 -04004727 if (count == 0)
4728 return 0;
4729
Amir Goldstein2846e822011-05-09 10:46:41 -04004730 ext4_get_group_no_and_offset(sb, block, &block_group, &bit);
Amir Goldstein2846e822011-05-09 10:46:41 -04004731 /*
4732 * Check to see if we are freeing blocks across a group
4733 * boundary.
4734 */
Yongqiang Yangcc7365d2011-07-26 21:46:07 -04004735 if (bit + count > EXT4_BLOCKS_PER_GROUP(sb)) {
4736 ext4_warning(sb, "too much blocks added to group %u\n",
4737 block_group);
4738 err = -EINVAL;
Amir Goldstein2846e822011-05-09 10:46:41 -04004739 goto error_return;
Yongqiang Yangcc7365d2011-07-26 21:46:07 -04004740 }
Theodore Ts'o2cd05cc2011-05-09 10:58:45 -04004741
Amir Goldstein2846e822011-05-09 10:46:41 -04004742 bitmap_bh = ext4_read_block_bitmap(sb, block_group);
Yongqiang Yangcc7365d2011-07-26 21:46:07 -04004743 if (!bitmap_bh) {
4744 err = -EIO;
Amir Goldstein2846e822011-05-09 10:46:41 -04004745 goto error_return;
Yongqiang Yangcc7365d2011-07-26 21:46:07 -04004746 }
4747
Amir Goldstein2846e822011-05-09 10:46:41 -04004748 desc = ext4_get_group_desc(sb, block_group, &gd_bh);
Yongqiang Yangcc7365d2011-07-26 21:46:07 -04004749 if (!desc) {
4750 err = -EIO;
Amir Goldstein2846e822011-05-09 10:46:41 -04004751 goto error_return;
Yongqiang Yangcc7365d2011-07-26 21:46:07 -04004752 }
Amir Goldstein2846e822011-05-09 10:46:41 -04004753
4754 if (in_range(ext4_block_bitmap(sb, desc), block, count) ||
4755 in_range(ext4_inode_bitmap(sb, desc), block, count) ||
4756 in_range(block, ext4_inode_table(sb, desc), sbi->s_itb_per_group) ||
4757 in_range(block + count - 1, ext4_inode_table(sb, desc),
4758 sbi->s_itb_per_group)) {
4759 ext4_error(sb, "Adding blocks in system zones - "
4760 "Block = %llu, count = %lu",
4761 block, count);
Yongqiang Yangcc7365d2011-07-26 21:46:07 -04004762 err = -EINVAL;
Amir Goldstein2846e822011-05-09 10:46:41 -04004763 goto error_return;
4764 }
4765
Theodore Ts'o2cd05cc2011-05-09 10:58:45 -04004766 BUFFER_TRACE(bitmap_bh, "getting write access");
4767 err = ext4_journal_get_write_access(handle, bitmap_bh);
Amir Goldstein2846e822011-05-09 10:46:41 -04004768 if (err)
4769 goto error_return;
4770
4771 /*
4772 * We are about to modify some metadata. Call the journal APIs
4773 * to unshare ->b_data if a currently-committing transaction is
4774 * using it
4775 */
4776 BUFFER_TRACE(gd_bh, "get_write_access");
4777 err = ext4_journal_get_write_access(handle, gd_bh);
4778 if (err)
4779 goto error_return;
Amir Goldsteine73a3472011-05-09 21:40:01 -04004780
Amir Goldstein2846e822011-05-09 10:46:41 -04004781 for (i = 0, blocks_freed = 0; i < count; i++) {
4782 BUFFER_TRACE(bitmap_bh, "clear bit");
Amir Goldsteine73a3472011-05-09 21:40:01 -04004783 if (!mb_test_bit(bit + i, bitmap_bh->b_data)) {
Amir Goldstein2846e822011-05-09 10:46:41 -04004784 ext4_error(sb, "bit already cleared for block %llu",
4785 (ext4_fsblk_t)(block + i));
4786 BUFFER_TRACE(bitmap_bh, "bit already cleared");
4787 } else {
4788 blocks_freed++;
4789 }
4790 }
Amir Goldsteine73a3472011-05-09 21:40:01 -04004791
4792 err = ext4_mb_load_buddy(sb, block_group, &e4b);
4793 if (err)
4794 goto error_return;
4795
4796 /*
4797 * need to update group_info->bb_free and bitmap
4798 * with group lock held. generate_buddy look at
4799 * them with group lock_held
4800 */
Amir Goldstein2846e822011-05-09 10:46:41 -04004801 ext4_lock_group(sb, block_group);
Amir Goldsteine73a3472011-05-09 21:40:01 -04004802 mb_clear_bits(bitmap_bh->b_data, bit, count);
4803 mb_free_blocks(NULL, &e4b, bit, count);
Theodore Ts'o021b65b2011-09-09 19:08:51 -04004804 blk_free_count = blocks_freed + ext4_free_group_clusters(sb, desc);
4805 ext4_free_group_clusters_set(sb, desc, blk_free_count);
Amir Goldstein2846e822011-05-09 10:46:41 -04004806 desc->bg_checksum = ext4_group_desc_csum(sbi, block_group, desc);
4807 ext4_unlock_group(sb, block_group);
Theodore Ts'o57042652011-09-09 18:56:51 -04004808 percpu_counter_add(&sbi->s_freeclusters_counter,
4809 EXT4_B2C(sbi, blocks_freed));
Amir Goldstein2846e822011-05-09 10:46:41 -04004810
4811 if (sbi->s_log_groups_per_flex) {
4812 ext4_group_t flex_group = ext4_flex_group(sbi, block_group);
Theodore Ts'o24aaa8e2011-09-09 18:58:51 -04004813 atomic_add(EXT4_B2C(sbi, blocks_freed),
4814 &sbi->s_flex_groups[flex_group].free_clusters);
Amir Goldstein2846e822011-05-09 10:46:41 -04004815 }
Amir Goldsteine73a3472011-05-09 21:40:01 -04004816
4817 ext4_mb_unload_buddy(&e4b);
Amir Goldstein2846e822011-05-09 10:46:41 -04004818
4819 /* We dirtied the bitmap block */
4820 BUFFER_TRACE(bitmap_bh, "dirtied bitmap block");
4821 err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh);
4822
4823 /* And the group descriptor block */
4824 BUFFER_TRACE(gd_bh, "dirtied group descriptor block");
4825 ret = ext4_handle_dirty_metadata(handle, NULL, gd_bh);
4826 if (!err)
4827 err = ret;
4828
4829error_return:
4830 brelse(bitmap_bh);
4831 ext4_std_error(sb, err);
Yongqiang Yangcc7365d2011-07-26 21:46:07 -04004832 return err;
Amir Goldstein2846e822011-05-09 10:46:41 -04004833}
4834
4835/**
Lukas Czerner7360d172010-10-27 21:30:12 -04004836 * ext4_trim_extent -- function to TRIM one single free extent in the group
4837 * @sb: super block for the file system
4838 * @start: starting block of the free extent in the alloc. group
4839 * @count: number of blocks to TRIM
4840 * @group: alloc. group we are working with
4841 * @e4b: ext4 buddy for the group
4842 *
4843 * Trim "count" blocks starting at "start" in the "group". To assure that no
4844 * one will allocate those blocks, mark it as used in buddy bitmap. This must
4845 * be called with under the group lock.
4846 */
Theodore Ts'od9f34502011-04-30 13:47:24 -04004847static void ext4_trim_extent(struct super_block *sb, int start, int count,
4848 ext4_group_t group, struct ext4_buddy *e4b)
Lukas Czerner7360d172010-10-27 21:30:12 -04004849{
4850 struct ext4_free_extent ex;
Lukas Czerner7360d172010-10-27 21:30:12 -04004851
Tao Mab3d4c2b2011-07-11 00:01:52 -04004852 trace_ext4_trim_extent(sb, group, start, count);
4853
Lukas Czerner7360d172010-10-27 21:30:12 -04004854 assert_spin_locked(ext4_group_lock_ptr(sb, group));
4855
4856 ex.fe_start = start;
4857 ex.fe_group = group;
4858 ex.fe_len = count;
4859
4860 /*
4861 * Mark blocks used, so no one can reuse them while
4862 * being trimmed.
4863 */
4864 mb_mark_used(e4b, &ex);
4865 ext4_unlock_group(sb, group);
Theodore Ts'od9f34502011-04-30 13:47:24 -04004866 ext4_issue_discard(sb, group, start, count);
Lukas Czerner7360d172010-10-27 21:30:12 -04004867 ext4_lock_group(sb, group);
4868 mb_free_blocks(NULL, e4b, start, ex.fe_len);
Lukas Czerner7360d172010-10-27 21:30:12 -04004869}
4870
4871/**
4872 * ext4_trim_all_free -- function to trim all free space in alloc. group
4873 * @sb: super block for file system
Tao Ma22612282011-07-11 00:04:34 -04004874 * @group: group to be trimmed
Lukas Czerner7360d172010-10-27 21:30:12 -04004875 * @start: first group block to examine
4876 * @max: last group block to examine
4877 * @minblocks: minimum extent block count
4878 *
4879 * ext4_trim_all_free walks through group's buddy bitmap searching for free
4880 * extents. When the free block is found, ext4_trim_extent is called to TRIM
4881 * the extent.
4882 *
4883 *
4884 * ext4_trim_all_free walks through group's block bitmap searching for free
4885 * extents. When the free extent is found, mark it as used in group buddy
4886 * bitmap. Then issue a TRIM command on this extent and free the extent in
4887 * the group buddy bitmap. This is done until whole group is scanned.
4888 */
Lukas Czerner0b75a842011-02-23 12:22:49 -05004889static ext4_grpblk_t
Lukas Czerner78944082011-05-24 18:16:27 -04004890ext4_trim_all_free(struct super_block *sb, ext4_group_t group,
4891 ext4_grpblk_t start, ext4_grpblk_t max,
4892 ext4_grpblk_t minblocks)
Lukas Czerner7360d172010-10-27 21:30:12 -04004893{
4894 void *bitmap;
Tao Ma169ddc32011-07-11 00:00:07 -04004895 ext4_grpblk_t next, count = 0, free_count = 0;
Lukas Czerner78944082011-05-24 18:16:27 -04004896 struct ext4_buddy e4b;
4897 int ret;
Lukas Czerner7360d172010-10-27 21:30:12 -04004898
Tao Mab3d4c2b2011-07-11 00:01:52 -04004899 trace_ext4_trim_all_free(sb, group, start, max);
4900
Lukas Czerner78944082011-05-24 18:16:27 -04004901 ret = ext4_mb_load_buddy(sb, group, &e4b);
4902 if (ret) {
4903 ext4_error(sb, "Error in loading buddy "
4904 "information for %u", group);
4905 return ret;
4906 }
Lukas Czerner78944082011-05-24 18:16:27 -04004907 bitmap = e4b.bd_bitmap;
Lukas Czerner28739ee2011-05-24 18:28:07 -04004908
4909 ext4_lock_group(sb, group);
Tao Ma3d56b8d2011-07-11 00:03:38 -04004910 if (EXT4_MB_GRP_WAS_TRIMMED(e4b.bd_info) &&
4911 minblocks >= atomic_read(&EXT4_SB(sb)->s_last_trim_minblks))
4912 goto out;
4913
Lukas Czerner78944082011-05-24 18:16:27 -04004914 start = (e4b.bd_info->bb_first_free > start) ?
4915 e4b.bd_info->bb_first_free : start;
Lukas Czerner7360d172010-10-27 21:30:12 -04004916
Lukas Czerner913eed832012-03-21 21:22:22 -04004917 while (start <= max) {
4918 start = mb_find_next_zero_bit(bitmap, max + 1, start);
4919 if (start > max)
Lukas Czerner7360d172010-10-27 21:30:12 -04004920 break;
Lukas Czerner913eed832012-03-21 21:22:22 -04004921 next = mb_find_next_bit(bitmap, max + 1, start);
Lukas Czerner7360d172010-10-27 21:30:12 -04004922
4923 if ((next - start) >= minblocks) {
Theodore Ts'od9f34502011-04-30 13:47:24 -04004924 ext4_trim_extent(sb, start,
Lukas Czerner78944082011-05-24 18:16:27 -04004925 next - start, group, &e4b);
Lukas Czerner7360d172010-10-27 21:30:12 -04004926 count += next - start;
4927 }
Tao Ma169ddc32011-07-11 00:00:07 -04004928 free_count += next - start;
Lukas Czerner7360d172010-10-27 21:30:12 -04004929 start = next + 1;
4930
4931 if (fatal_signal_pending(current)) {
4932 count = -ERESTARTSYS;
4933 break;
4934 }
4935
4936 if (need_resched()) {
4937 ext4_unlock_group(sb, group);
4938 cond_resched();
4939 ext4_lock_group(sb, group);
4940 }
4941
Tao Ma169ddc32011-07-11 00:00:07 -04004942 if ((e4b.bd_info->bb_free - free_count) < minblocks)
Lukas Czerner7360d172010-10-27 21:30:12 -04004943 break;
4944 }
Tao Ma3d56b8d2011-07-11 00:03:38 -04004945
4946 if (!ret)
4947 EXT4_MB_GRP_SET_TRIMMED(e4b.bd_info);
4948out:
Lukas Czerner7360d172010-10-27 21:30:12 -04004949 ext4_unlock_group(sb, group);
Lukas Czerner78944082011-05-24 18:16:27 -04004950 ext4_mb_unload_buddy(&e4b);
Lukas Czerner7360d172010-10-27 21:30:12 -04004951
4952 ext4_debug("trimmed %d blocks in the group %d\n",
4953 count, group);
4954
Lukas Czerner7360d172010-10-27 21:30:12 -04004955 return count;
4956}
4957
4958/**
4959 * ext4_trim_fs() -- trim ioctl handle function
4960 * @sb: superblock for filesystem
4961 * @range: fstrim_range structure
4962 *
4963 * start: First Byte to trim
4964 * len: number of Bytes to trim from start
4965 * minlen: minimum extent length in Bytes
4966 * ext4_trim_fs goes through all allocation groups containing Bytes from
4967 * start to start+len. For each such a group ext4_trim_all_free function
4968 * is invoked to trim all free space.
4969 */
4970int ext4_trim_fs(struct super_block *sb, struct fstrim_range *range)
4971{
Lukas Czerner78944082011-05-24 18:16:27 -04004972 struct ext4_group_info *grp;
Lukas Czerner913eed832012-03-21 21:22:22 -04004973 ext4_group_t group, first_group, last_group;
Theodore Ts'o7137d7a2011-09-09 18:38:51 -04004974 ext4_grpblk_t cnt = 0, first_cluster, last_cluster;
Lukas Czerner913eed832012-03-21 21:22:22 -04004975 uint64_t start, end, minlen, trimmed = 0;
Jan Kara0f0a25b2011-01-11 15:16:31 -05004976 ext4_fsblk_t first_data_blk =
4977 le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block);
Lukas Czerner913eed832012-03-21 21:22:22 -04004978 ext4_fsblk_t max_blks = ext4_blocks_count(EXT4_SB(sb)->s_es);
Lukas Czerner7360d172010-10-27 21:30:12 -04004979 int ret = 0;
4980
4981 start = range->start >> sb->s_blocksize_bits;
Lukas Czerner913eed832012-03-21 21:22:22 -04004982 end = start + (range->len >> sb->s_blocksize_bits) - 1;
Lukas Czerner7360d172010-10-27 21:30:12 -04004983 minlen = range->minlen >> sb->s_blocksize_bits;
Lukas Czerner7360d172010-10-27 21:30:12 -04004984
Lukas Czerner913eed832012-03-21 21:22:22 -04004985 if (unlikely(minlen > EXT4_CLUSTERS_PER_GROUP(sb)) ||
4986 unlikely(start >= max_blks))
Lukas Czerner7360d172010-10-27 21:30:12 -04004987 return -EINVAL;
Lukas Czerner913eed832012-03-21 21:22:22 -04004988 if (end >= max_blks)
4989 end = max_blks - 1;
4990 if (end <= first_data_blk)
Tao Ma22f10452011-07-10 23:52:37 -04004991 goto out;
Lukas Czerner913eed832012-03-21 21:22:22 -04004992 if (start < first_data_blk)
Jan Kara0f0a25b2011-01-11 15:16:31 -05004993 start = first_data_blk;
Lukas Czerner7360d172010-10-27 21:30:12 -04004994
Lukas Czerner913eed832012-03-21 21:22:22 -04004995 /* Determine first and last group to examine based on start and end */
Lukas Czerner7360d172010-10-27 21:30:12 -04004996 ext4_get_group_no_and_offset(sb, (ext4_fsblk_t) start,
Theodore Ts'o7137d7a2011-09-09 18:38:51 -04004997 &first_group, &first_cluster);
Lukas Czerner913eed832012-03-21 21:22:22 -04004998 ext4_get_group_no_and_offset(sb, (ext4_fsblk_t) end,
Theodore Ts'o7137d7a2011-09-09 18:38:51 -04004999 &last_group, &last_cluster);
Lukas Czerner7360d172010-10-27 21:30:12 -04005000
Lukas Czerner913eed832012-03-21 21:22:22 -04005001 /* end now represents the last cluster to discard in this group */
5002 end = EXT4_CLUSTERS_PER_GROUP(sb) - 1;
Lukas Czerner7360d172010-10-27 21:30:12 -04005003
5004 for (group = first_group; group <= last_group; group++) {
Lukas Czerner78944082011-05-24 18:16:27 -04005005 grp = ext4_get_group_info(sb, group);
5006 /* We only do this if the grp has never been initialized */
5007 if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) {
5008 ret = ext4_mb_init_group(sb, group);
5009 if (ret)
5010 break;
Lukas Czerner7360d172010-10-27 21:30:12 -04005011 }
5012
Tao Ma0ba08512011-03-23 15:48:11 -04005013 /*
Lukas Czerner913eed832012-03-21 21:22:22 -04005014 * For all the groups except the last one, last cluster will
5015 * always be EXT4_CLUSTERS_PER_GROUP(sb)-1, so we only need to
5016 * change it for the last group, note that last_cluster is
5017 * already computed earlier by ext4_get_group_no_and_offset()
Tao Ma0ba08512011-03-23 15:48:11 -04005018 */
Lukas Czerner913eed832012-03-21 21:22:22 -04005019 if (group == last_group)
5020 end = last_cluster;
Lukas Czerner7360d172010-10-27 21:30:12 -04005021
Lukas Czerner78944082011-05-24 18:16:27 -04005022 if (grp->bb_free >= minlen) {
Theodore Ts'o7137d7a2011-09-09 18:38:51 -04005023 cnt = ext4_trim_all_free(sb, group, first_cluster,
Lukas Czerner913eed832012-03-21 21:22:22 -04005024 end, minlen);
Lukas Czerner7360d172010-10-27 21:30:12 -04005025 if (cnt < 0) {
5026 ret = cnt;
Lukas Czerner7360d172010-10-27 21:30:12 -04005027 break;
5028 }
Lukas Czerner21e7fd22012-03-21 21:24:22 -04005029 trimmed += cnt;
Lukas Czerner7360d172010-10-27 21:30:12 -04005030 }
Lukas Czerner913eed832012-03-21 21:22:22 -04005031
5032 /*
5033 * For every group except the first one, we are sure
5034 * that the first cluster to discard will be cluster #0.
5035 */
Theodore Ts'o7137d7a2011-09-09 18:38:51 -04005036 first_cluster = 0;
Lukas Czerner7360d172010-10-27 21:30:12 -04005037 }
Lukas Czerner7360d172010-10-27 21:30:12 -04005038
Tao Ma3d56b8d2011-07-11 00:03:38 -04005039 if (!ret)
5040 atomic_set(&EXT4_SB(sb)->s_last_trim_minblks, minlen);
5041
Tao Ma22f10452011-07-10 23:52:37 -04005042out:
Lukas Czernera7967f02012-03-21 21:26:22 -04005043 range->len = trimmed * sb->s_blocksize;
Lukas Czerner7360d172010-10-27 21:30:12 -04005044 return ret;
5045}