blob: a36a8e3b703fdea9090c9c981cbf21410412f345 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Nathan Scott7b718762005-11-02 14:58:39 +11002 * Copyright (c) 2000-2005 Silicon Graphics, Inc.
3 * All Rights Reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
Nathan Scott7b718762005-11-02 14:58:39 +11005 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License as
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 * published by the Free Software Foundation.
8 *
Nathan Scott7b718762005-11-02 14:58:39 +11009 * This program is distributed in the hope that it would be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 *
Nathan Scott7b718762005-11-02 14:58:39 +110014 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
Linus Torvalds1da177e2005-04-16 15:20:36 -070017 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/stddef.h>
19#include <linux/errno.h>
20#include <linux/slab.h>
21#include <linux/pagemap.h>
22#include <linux/init.h>
23#include <linux/vmalloc.h>
24#include <linux/bio.h>
25#include <linux/sysctl.h>
26#include <linux/proc_fs.h>
27#include <linux/workqueue.h>
28#include <linux/percpu.h>
29#include <linux/blkdev.h>
30#include <linux/hash.h>
Christoph Hellwig4df08c52005-09-05 08:34:18 +100031#include <linux/kthread.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070032#include "xfs_linux.h"
33
Nathan Scottce8e9222006-01-11 15:39:08 +110034STATIC kmem_zone_t *xfs_buf_zone;
35STATIC kmem_shaker_t xfs_buf_shake;
David Chinnera6867a62006-01-11 15:37:58 +110036STATIC int xfsbufd(void *);
Al Viro27496a82005-10-21 03:20:48 -040037STATIC int xfsbufd_wakeup(int, gfp_t);
Nathan Scottce8e9222006-01-11 15:39:08 +110038STATIC void xfs_buf_delwri_queue(xfs_buf_t *, int);
Christoph Hellwig23ea4032005-06-21 15:14:01 +100039
40STATIC struct workqueue_struct *xfslogd_workqueue;
Christoph Hellwig0829c362005-09-02 16:58:49 +100041struct workqueue_struct *xfsdatad_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070042
Nathan Scottce8e9222006-01-11 15:39:08 +110043#ifdef XFS_BUF_TRACE
Linus Torvalds1da177e2005-04-16 15:20:36 -070044void
Nathan Scottce8e9222006-01-11 15:39:08 +110045xfs_buf_trace(
46 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -070047 char *id,
48 void *data,
49 void *ra)
50{
Nathan Scottce8e9222006-01-11 15:39:08 +110051 ktrace_enter(xfs_buf_trace_buf,
52 bp, id,
53 (void *)(unsigned long)bp->b_flags,
54 (void *)(unsigned long)bp->b_hold.counter,
55 (void *)(unsigned long)bp->b_sema.count.counter,
Linus Torvalds1da177e2005-04-16 15:20:36 -070056 (void *)current,
57 data, ra,
Nathan Scottce8e9222006-01-11 15:39:08 +110058 (void *)(unsigned long)((bp->b_file_offset>>32) & 0xffffffff),
59 (void *)(unsigned long)(bp->b_file_offset & 0xffffffff),
60 (void *)(unsigned long)bp->b_buffer_length,
Linus Torvalds1da177e2005-04-16 15:20:36 -070061 NULL, NULL, NULL, NULL, NULL);
62}
Nathan Scottce8e9222006-01-11 15:39:08 +110063ktrace_t *xfs_buf_trace_buf;
64#define XFS_BUF_TRACE_SIZE 4096
65#define XB_TRACE(bp, id, data) \
66 xfs_buf_trace(bp, id, (void *)data, (void *)__builtin_return_address(0))
Linus Torvalds1da177e2005-04-16 15:20:36 -070067#else
Nathan Scottce8e9222006-01-11 15:39:08 +110068#define XB_TRACE(bp, id, data) do { } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -070069#endif
70
Nathan Scottce8e9222006-01-11 15:39:08 +110071#ifdef XFS_BUF_LOCK_TRACKING
72# define XB_SET_OWNER(bp) ((bp)->b_last_holder = current->pid)
73# define XB_CLEAR_OWNER(bp) ((bp)->b_last_holder = -1)
74# define XB_GET_OWNER(bp) ((bp)->b_last_holder)
Linus Torvalds1da177e2005-04-16 15:20:36 -070075#else
Nathan Scottce8e9222006-01-11 15:39:08 +110076# define XB_SET_OWNER(bp) do { } while (0)
77# define XB_CLEAR_OWNER(bp) do { } while (0)
78# define XB_GET_OWNER(bp) do { } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -070079#endif
80
Nathan Scottce8e9222006-01-11 15:39:08 +110081#define xb_to_gfp(flags) \
82 ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : \
83 ((flags) & XBF_DONT_BLOCK) ? GFP_NOFS : GFP_KERNEL) | __GFP_NOWARN)
Linus Torvalds1da177e2005-04-16 15:20:36 -070084
Nathan Scottce8e9222006-01-11 15:39:08 +110085#define xb_to_km(flags) \
86 (((flags) & XBF_DONT_BLOCK) ? KM_NOFS : KM_SLEEP)
Linus Torvalds1da177e2005-04-16 15:20:36 -070087
Nathan Scottce8e9222006-01-11 15:39:08 +110088#define xfs_buf_allocate(flags) \
89 kmem_zone_alloc(xfs_buf_zone, xb_to_km(flags))
90#define xfs_buf_deallocate(bp) \
91 kmem_zone_free(xfs_buf_zone, (bp));
Linus Torvalds1da177e2005-04-16 15:20:36 -070092
93/*
Nathan Scottce8e9222006-01-11 15:39:08 +110094 * Page Region interfaces.
Linus Torvalds1da177e2005-04-16 15:20:36 -070095 *
Nathan Scottce8e9222006-01-11 15:39:08 +110096 * For pages in filesystems where the blocksize is smaller than the
97 * pagesize, we use the page->private field (long) to hold a bitmap
98 * of uptodate regions within the page.
Linus Torvalds1da177e2005-04-16 15:20:36 -070099 *
Nathan Scottce8e9222006-01-11 15:39:08 +1100100 * Each such region is "bytes per page / bits per long" bytes long.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101 *
Nathan Scottce8e9222006-01-11 15:39:08 +1100102 * NBPPR == number-of-bytes-per-page-region
103 * BTOPR == bytes-to-page-region (rounded up)
104 * BTOPRT == bytes-to-page-region-truncated (rounded down)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105 */
106#if (BITS_PER_LONG == 32)
107#define PRSHIFT (PAGE_CACHE_SHIFT - 5) /* (32 == 1<<5) */
108#elif (BITS_PER_LONG == 64)
109#define PRSHIFT (PAGE_CACHE_SHIFT - 6) /* (64 == 1<<6) */
110#else
111#error BITS_PER_LONG must be 32 or 64
112#endif
113#define NBPPR (PAGE_CACHE_SIZE/BITS_PER_LONG)
114#define BTOPR(b) (((unsigned int)(b) + (NBPPR - 1)) >> PRSHIFT)
115#define BTOPRT(b) (((unsigned int)(b) >> PRSHIFT))
116
117STATIC unsigned long
118page_region_mask(
119 size_t offset,
120 size_t length)
121{
122 unsigned long mask;
123 int first, final;
124
125 first = BTOPR(offset);
126 final = BTOPRT(offset + length - 1);
127 first = min(first, final);
128
129 mask = ~0UL;
130 mask <<= BITS_PER_LONG - (final - first);
131 mask >>= BITS_PER_LONG - (final);
132
133 ASSERT(offset + length <= PAGE_CACHE_SIZE);
134 ASSERT((final - first) < BITS_PER_LONG && (final - first) >= 0);
135
136 return mask;
137}
138
139STATIC inline void
140set_page_region(
141 struct page *page,
142 size_t offset,
143 size_t length)
144{
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700145 set_page_private(page,
146 page_private(page) | page_region_mask(offset, length));
147 if (page_private(page) == ~0UL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700148 SetPageUptodate(page);
149}
150
151STATIC inline int
152test_page_region(
153 struct page *page,
154 size_t offset,
155 size_t length)
156{
157 unsigned long mask = page_region_mask(offset, length);
158
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700159 return (mask && (page_private(page) & mask) == mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160}
161
162/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100163 * Mapping of multi-page buffers into contiguous virtual space
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164 */
165
166typedef struct a_list {
167 void *vm_addr;
168 struct a_list *next;
169} a_list_t;
170
171STATIC a_list_t *as_free_head;
172STATIC int as_list_len;
173STATIC DEFINE_SPINLOCK(as_lock);
174
175/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100176 * Try to batch vunmaps because they are costly.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177 */
178STATIC void
179free_address(
180 void *addr)
181{
182 a_list_t *aentry;
183
184 aentry = kmalloc(sizeof(a_list_t), GFP_ATOMIC & ~__GFP_HIGH);
185 if (likely(aentry)) {
186 spin_lock(&as_lock);
187 aentry->next = as_free_head;
188 aentry->vm_addr = addr;
189 as_free_head = aentry;
190 as_list_len++;
191 spin_unlock(&as_lock);
192 } else {
193 vunmap(addr);
194 }
195}
196
197STATIC void
198purge_addresses(void)
199{
200 a_list_t *aentry, *old;
201
202 if (as_free_head == NULL)
203 return;
204
205 spin_lock(&as_lock);
206 aentry = as_free_head;
207 as_free_head = NULL;
208 as_list_len = 0;
209 spin_unlock(&as_lock);
210
211 while ((old = aentry) != NULL) {
212 vunmap(aentry->vm_addr);
213 aentry = aentry->next;
214 kfree(old);
215 }
216}
217
218/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100219 * Internal xfs_buf_t object manipulation
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220 */
221
222STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +1100223_xfs_buf_initialize(
224 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700225 xfs_buftarg_t *target,
Nathan Scott204ab252006-01-11 20:50:22 +1100226 xfs_off_t range_base,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700227 size_t range_length,
Nathan Scottce8e9222006-01-11 15:39:08 +1100228 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700229{
230 /*
Nathan Scottce8e9222006-01-11 15:39:08 +1100231 * We don't want certain flags to appear in b_flags.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700232 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100233 flags &= ~(XBF_LOCK|XBF_MAPPED|XBF_DONT_BLOCK|XBF_READ_AHEAD);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234
Nathan Scottce8e9222006-01-11 15:39:08 +1100235 memset(bp, 0, sizeof(xfs_buf_t));
236 atomic_set(&bp->b_hold, 1);
237 init_MUTEX_LOCKED(&bp->b_iodonesema);
238 INIT_LIST_HEAD(&bp->b_list);
239 INIT_LIST_HEAD(&bp->b_hash_list);
240 init_MUTEX_LOCKED(&bp->b_sema); /* held, no waiters */
241 XB_SET_OWNER(bp);
242 bp->b_target = target;
243 bp->b_file_offset = range_base;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244 /*
245 * Set buffer_length and count_desired to the same value initially.
246 * I/O routines should use count_desired, which will be the same in
247 * most cases but may be reset (e.g. XFS recovery).
248 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100249 bp->b_buffer_length = bp->b_count_desired = range_length;
250 bp->b_flags = flags;
251 bp->b_bn = XFS_BUF_DADDR_NULL;
252 atomic_set(&bp->b_pin_count, 0);
253 init_waitqueue_head(&bp->b_waiters);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254
Nathan Scottce8e9222006-01-11 15:39:08 +1100255 XFS_STATS_INC(xb_create);
256 XB_TRACE(bp, "initialize", target);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700257}
258
259/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100260 * Allocate a page array capable of holding a specified number
261 * of pages, and point the page buf at it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700262 */
263STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100264_xfs_buf_get_pages(
265 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266 int page_count,
Nathan Scottce8e9222006-01-11 15:39:08 +1100267 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268{
269 /* Make sure that we have a page list */
Nathan Scottce8e9222006-01-11 15:39:08 +1100270 if (bp->b_pages == NULL) {
271 bp->b_offset = xfs_buf_poff(bp->b_file_offset);
272 bp->b_page_count = page_count;
273 if (page_count <= XB_PAGES) {
274 bp->b_pages = bp->b_page_array;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100276 bp->b_pages = kmem_alloc(sizeof(struct page *) *
277 page_count, xb_to_km(flags));
278 if (bp->b_pages == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279 return -ENOMEM;
280 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100281 memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282 }
283 return 0;
284}
285
286/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100287 * Frees b_pages if it was allocated.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700288 */
289STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +1100290_xfs_buf_free_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291 xfs_buf_t *bp)
292{
Nathan Scottce8e9222006-01-11 15:39:08 +1100293 if (bp->b_pages != bp->b_page_array) {
294 kmem_free(bp->b_pages,
295 bp->b_page_count * sizeof(struct page *));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700296 }
297}
298
299/*
300 * Releases the specified buffer.
301 *
302 * The modification state of any associated pages is left unchanged.
Nathan Scottce8e9222006-01-11 15:39:08 +1100303 * The buffer most not be on any hash - use xfs_buf_rele instead for
Linus Torvalds1da177e2005-04-16 15:20:36 -0700304 * hashed and refcounted buffers
305 */
306void
Nathan Scottce8e9222006-01-11 15:39:08 +1100307xfs_buf_free(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308 xfs_buf_t *bp)
309{
Nathan Scottce8e9222006-01-11 15:39:08 +1100310 XB_TRACE(bp, "free", 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311
Nathan Scottce8e9222006-01-11 15:39:08 +1100312 ASSERT(list_empty(&bp->b_hash_list));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313
Nathan Scottce8e9222006-01-11 15:39:08 +1100314 if (bp->b_flags & _XBF_PAGE_CACHE) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700315 uint i;
316
Nathan Scottce8e9222006-01-11 15:39:08 +1100317 if ((bp->b_flags & XBF_MAPPED) && (bp->b_page_count > 1))
318 free_address(bp->b_addr - bp->b_offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700319
Nathan Scottce8e9222006-01-11 15:39:08 +1100320 for (i = 0; i < bp->b_page_count; i++)
321 page_cache_release(bp->b_pages[i]);
322 _xfs_buf_free_pages(bp);
323 } else if (bp->b_flags & _XBF_KMEM_ALLOC) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700324 /*
Nathan Scottce8e9222006-01-11 15:39:08 +1100325 * XXX(hch): bp->b_count_desired might be incorrect (see
326 * xfs_buf_associate_memory for details), but fortunately
Linus Torvalds1da177e2005-04-16 15:20:36 -0700327 * the Linux version of kmem_free ignores the len argument..
328 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100329 kmem_free(bp->b_addr, bp->b_count_desired);
330 _xfs_buf_free_pages(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331 }
332
Nathan Scottce8e9222006-01-11 15:39:08 +1100333 xfs_buf_deallocate(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700334}
335
336/*
337 * Finds all pages for buffer in question and builds it's page list.
338 */
339STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100340_xfs_buf_lookup_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700341 xfs_buf_t *bp,
342 uint flags)
343{
Nathan Scottce8e9222006-01-11 15:39:08 +1100344 struct address_space *mapping = bp->b_target->bt_mapping;
345 size_t blocksize = bp->b_target->bt_bsize;
346 size_t size = bp->b_count_desired;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347 size_t nbytes, offset;
Nathan Scottce8e9222006-01-11 15:39:08 +1100348 gfp_t gfp_mask = xb_to_gfp(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700349 unsigned short page_count, i;
350 pgoff_t first;
Nathan Scott204ab252006-01-11 20:50:22 +1100351 xfs_off_t end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700352 int error;
353
Nathan Scottce8e9222006-01-11 15:39:08 +1100354 end = bp->b_file_offset + bp->b_buffer_length;
355 page_count = xfs_buf_btoc(end) - xfs_buf_btoct(bp->b_file_offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700356
Nathan Scottce8e9222006-01-11 15:39:08 +1100357 error = _xfs_buf_get_pages(bp, page_count, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358 if (unlikely(error))
359 return error;
Nathan Scottce8e9222006-01-11 15:39:08 +1100360 bp->b_flags |= _XBF_PAGE_CACHE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361
Nathan Scottce8e9222006-01-11 15:39:08 +1100362 offset = bp->b_offset;
363 first = bp->b_file_offset >> PAGE_CACHE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364
Nathan Scottce8e9222006-01-11 15:39:08 +1100365 for (i = 0; i < bp->b_page_count; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700366 struct page *page;
367 uint retries = 0;
368
369 retry:
370 page = find_or_create_page(mapping, first + i, gfp_mask);
371 if (unlikely(page == NULL)) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100372 if (flags & XBF_READ_AHEAD) {
373 bp->b_page_count = i;
374 for (i = 0; i < bp->b_page_count; i++)
375 unlock_page(bp->b_pages[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376 return -ENOMEM;
377 }
378
379 /*
380 * This could deadlock.
381 *
382 * But until all the XFS lowlevel code is revamped to
383 * handle buffer allocation failures we can't do much.
384 */
385 if (!(++retries % 100))
386 printk(KERN_ERR
387 "XFS: possible memory allocation "
388 "deadlock in %s (mode:0x%x)\n",
389 __FUNCTION__, gfp_mask);
390
Nathan Scottce8e9222006-01-11 15:39:08 +1100391 XFS_STATS_INC(xb_page_retries);
Christoph Hellwig23ea4032005-06-21 15:14:01 +1000392 xfsbufd_wakeup(0, gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393 blk_congestion_wait(WRITE, HZ/50);
394 goto retry;
395 }
396
Nathan Scottce8e9222006-01-11 15:39:08 +1100397 XFS_STATS_INC(xb_page_found);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398
399 nbytes = min_t(size_t, size, PAGE_CACHE_SIZE - offset);
400 size -= nbytes;
401
402 if (!PageUptodate(page)) {
403 page_count--;
404 if (blocksize >= PAGE_CACHE_SIZE) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100405 if (flags & XBF_READ)
406 bp->b_locked = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407 } else if (!PagePrivate(page)) {
408 if (test_page_region(page, offset, nbytes))
409 page_count++;
410 }
411 }
412
Nathan Scottce8e9222006-01-11 15:39:08 +1100413 bp->b_pages[i] = page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700414 offset = 0;
415 }
416
Nathan Scottce8e9222006-01-11 15:39:08 +1100417 if (!bp->b_locked) {
418 for (i = 0; i < bp->b_page_count; i++)
419 unlock_page(bp->b_pages[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420 }
421
Nathan Scottce8e9222006-01-11 15:39:08 +1100422 if (page_count == bp->b_page_count)
423 bp->b_flags |= XBF_DONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424
Nathan Scottce8e9222006-01-11 15:39:08 +1100425 XB_TRACE(bp, "lookup_pages", (long)page_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426 return error;
427}
428
429/*
430 * Map buffer into kernel address-space if nessecary.
431 */
432STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100433_xfs_buf_map_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434 xfs_buf_t *bp,
435 uint flags)
436{
437 /* A single page buffer is always mappable */
Nathan Scottce8e9222006-01-11 15:39:08 +1100438 if (bp->b_page_count == 1) {
439 bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
440 bp->b_flags |= XBF_MAPPED;
441 } else if (flags & XBF_MAPPED) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700442 if (as_list_len > 64)
443 purge_addresses();
Nathan Scottce8e9222006-01-11 15:39:08 +1100444 bp->b_addr = vmap(bp->b_pages, bp->b_page_count,
445 VM_MAP, PAGE_KERNEL);
446 if (unlikely(bp->b_addr == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447 return -ENOMEM;
Nathan Scottce8e9222006-01-11 15:39:08 +1100448 bp->b_addr += bp->b_offset;
449 bp->b_flags |= XBF_MAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700450 }
451
452 return 0;
453}
454
455/*
456 * Finding and Reading Buffers
457 */
458
459/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100460 * Look up, and creates if absent, a lockable buffer for
Linus Torvalds1da177e2005-04-16 15:20:36 -0700461 * a given range of an inode. The buffer is returned
462 * locked. If other overlapping buffers exist, they are
463 * released before the new buffer is created and locked,
464 * which may imply that this call will block until those buffers
465 * are unlocked. No I/O is implied by this call.
466 */
467xfs_buf_t *
Nathan Scottce8e9222006-01-11 15:39:08 +1100468_xfs_buf_find(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469 xfs_buftarg_t *btp, /* block device target */
Nathan Scott204ab252006-01-11 20:50:22 +1100470 xfs_off_t ioff, /* starting offset of range */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471 size_t isize, /* length of range */
Nathan Scottce8e9222006-01-11 15:39:08 +1100472 xfs_buf_flags_t flags,
473 xfs_buf_t *new_bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474{
Nathan Scott204ab252006-01-11 20:50:22 +1100475 xfs_off_t range_base;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700476 size_t range_length;
477 xfs_bufhash_t *hash;
Nathan Scottce8e9222006-01-11 15:39:08 +1100478 xfs_buf_t *bp, *n;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700479
480 range_base = (ioff << BBSHIFT);
481 range_length = (isize << BBSHIFT);
482
483 /* Check for IOs smaller than the sector size / not sector aligned */
Nathan Scottce8e9222006-01-11 15:39:08 +1100484 ASSERT(!(range_length < (1 << btp->bt_sshift)));
Nathan Scott204ab252006-01-11 20:50:22 +1100485 ASSERT(!(range_base & (xfs_off_t)btp->bt_smask));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486
487 hash = &btp->bt_hash[hash_long((unsigned long)ioff, btp->bt_hashshift)];
488
489 spin_lock(&hash->bh_lock);
490
Nathan Scottce8e9222006-01-11 15:39:08 +1100491 list_for_each_entry_safe(bp, n, &hash->bh_list, b_hash_list) {
492 ASSERT(btp == bp->b_target);
493 if (bp->b_file_offset == range_base &&
494 bp->b_buffer_length == range_length) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700495 /*
Nathan Scottce8e9222006-01-11 15:39:08 +1100496 * If we look at something, bring it to the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497 * front of the list for next time.
498 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100499 atomic_inc(&bp->b_hold);
500 list_move(&bp->b_hash_list, &hash->bh_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700501 goto found;
502 }
503 }
504
505 /* No match found */
Nathan Scottce8e9222006-01-11 15:39:08 +1100506 if (new_bp) {
507 _xfs_buf_initialize(new_bp, btp, range_base,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700508 range_length, flags);
Nathan Scottce8e9222006-01-11 15:39:08 +1100509 new_bp->b_hash = hash;
510 list_add(&new_bp->b_hash_list, &hash->bh_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700511 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100512 XFS_STATS_INC(xb_miss_locked);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513 }
514
515 spin_unlock(&hash->bh_lock);
Nathan Scottce8e9222006-01-11 15:39:08 +1100516 return new_bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700517
518found:
519 spin_unlock(&hash->bh_lock);
520
521 /* Attempt to get the semaphore without sleeping,
522 * if this does not work then we need to drop the
523 * spinlock and do a hard attempt on the semaphore.
524 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100525 if (down_trylock(&bp->b_sema)) {
526 if (!(flags & XBF_TRYLOCK)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700527 /* wait for buffer ownership */
Nathan Scottce8e9222006-01-11 15:39:08 +1100528 XB_TRACE(bp, "get_lock", 0);
529 xfs_buf_lock(bp);
530 XFS_STATS_INC(xb_get_locked_waited);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700531 } else {
532 /* We asked for a trylock and failed, no need
533 * to look at file offset and length here, we
Nathan Scottce8e9222006-01-11 15:39:08 +1100534 * know that this buffer at least overlaps our
535 * buffer and is locked, therefore our buffer
536 * either does not exist, or is this buffer.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100538 xfs_buf_rele(bp);
539 XFS_STATS_INC(xb_busy_locked);
540 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700541 }
542 } else {
543 /* trylock worked */
Nathan Scottce8e9222006-01-11 15:39:08 +1100544 XB_SET_OWNER(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545 }
546
Nathan Scottce8e9222006-01-11 15:39:08 +1100547 if (bp->b_flags & XBF_STALE) {
548 ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
549 bp->b_flags &= XBF_MAPPED;
David Chinner2f926582005-09-05 08:33:35 +1000550 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100551 XB_TRACE(bp, "got_lock", 0);
552 XFS_STATS_INC(xb_get_locked);
553 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554}
555
556/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100557 * Assembles a buffer covering the specified range.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700558 * Storage in memory for all portions of the buffer will be allocated,
559 * although backing storage may not be.
560 */
561xfs_buf_t *
Nathan Scottce8e9222006-01-11 15:39:08 +1100562xfs_buf_get_flags(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563 xfs_buftarg_t *target,/* target for buffer */
Nathan Scott204ab252006-01-11 20:50:22 +1100564 xfs_off_t ioff, /* starting offset of range */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565 size_t isize, /* length of range */
Nathan Scottce8e9222006-01-11 15:39:08 +1100566 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700567{
Nathan Scottce8e9222006-01-11 15:39:08 +1100568 xfs_buf_t *bp, *new_bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569 int error = 0, i;
570
Nathan Scottce8e9222006-01-11 15:39:08 +1100571 new_bp = xfs_buf_allocate(flags);
572 if (unlikely(!new_bp))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700573 return NULL;
574
Nathan Scottce8e9222006-01-11 15:39:08 +1100575 bp = _xfs_buf_find(target, ioff, isize, flags, new_bp);
576 if (bp == new_bp) {
577 error = _xfs_buf_lookup_pages(bp, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578 if (error)
579 goto no_buffer;
580 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100581 xfs_buf_deallocate(new_bp);
582 if (unlikely(bp == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700583 return NULL;
584 }
585
Nathan Scottce8e9222006-01-11 15:39:08 +1100586 for (i = 0; i < bp->b_page_count; i++)
587 mark_page_accessed(bp->b_pages[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700588
Nathan Scottce8e9222006-01-11 15:39:08 +1100589 if (!(bp->b_flags & XBF_MAPPED)) {
590 error = _xfs_buf_map_pages(bp, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700591 if (unlikely(error)) {
592 printk(KERN_WARNING "%s: failed to map pages\n",
593 __FUNCTION__);
594 goto no_buffer;
595 }
596 }
597
Nathan Scottce8e9222006-01-11 15:39:08 +1100598 XFS_STATS_INC(xb_get);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700599
600 /*
601 * Always fill in the block number now, the mapped cases can do
602 * their own overlay of this later.
603 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100604 bp->b_bn = ioff;
605 bp->b_count_desired = bp->b_buffer_length;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606
Nathan Scottce8e9222006-01-11 15:39:08 +1100607 XB_TRACE(bp, "get", (unsigned long)flags);
608 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700609
610 no_buffer:
Nathan Scottce8e9222006-01-11 15:39:08 +1100611 if (flags & (XBF_LOCK | XBF_TRYLOCK))
612 xfs_buf_unlock(bp);
613 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700614 return NULL;
615}
616
617xfs_buf_t *
618xfs_buf_read_flags(
619 xfs_buftarg_t *target,
Nathan Scott204ab252006-01-11 20:50:22 +1100620 xfs_off_t ioff,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700621 size_t isize,
Nathan Scottce8e9222006-01-11 15:39:08 +1100622 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623{
Nathan Scottce8e9222006-01-11 15:39:08 +1100624 xfs_buf_t *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700625
Nathan Scottce8e9222006-01-11 15:39:08 +1100626 flags |= XBF_READ;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627
Nathan Scottce8e9222006-01-11 15:39:08 +1100628 bp = xfs_buf_get_flags(target, ioff, isize, flags);
629 if (bp) {
630 if (!XFS_BUF_ISDONE(bp)) {
631 XB_TRACE(bp, "read", (unsigned long)flags);
632 XFS_STATS_INC(xb_get_read);
633 xfs_buf_iostart(bp, flags);
634 } else if (flags & XBF_ASYNC) {
635 XB_TRACE(bp, "read_async", (unsigned long)flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700636 /*
637 * Read ahead call which is already satisfied,
638 * drop the buffer
639 */
640 goto no_buffer;
641 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100642 XB_TRACE(bp, "read_done", (unsigned long)flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700643 /* We do not want read in the flags */
Nathan Scottce8e9222006-01-11 15:39:08 +1100644 bp->b_flags &= ~XBF_READ;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700645 }
646 }
647
Nathan Scottce8e9222006-01-11 15:39:08 +1100648 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700649
650 no_buffer:
Nathan Scottce8e9222006-01-11 15:39:08 +1100651 if (flags & (XBF_LOCK | XBF_TRYLOCK))
652 xfs_buf_unlock(bp);
653 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654 return NULL;
655}
656
657/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100658 * If we are not low on memory then do the readahead in a deadlock
659 * safe manner.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700660 */
661void
Nathan Scottce8e9222006-01-11 15:39:08 +1100662xfs_buf_readahead(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700663 xfs_buftarg_t *target,
Nathan Scott204ab252006-01-11 20:50:22 +1100664 xfs_off_t ioff,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700665 size_t isize,
Nathan Scottce8e9222006-01-11 15:39:08 +1100666 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700667{
668 struct backing_dev_info *bdi;
669
Nathan Scottce8e9222006-01-11 15:39:08 +1100670 bdi = target->bt_mapping->backing_dev_info;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671 if (bdi_read_congested(bdi))
672 return;
673
Nathan Scottce8e9222006-01-11 15:39:08 +1100674 flags |= (XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700675 xfs_buf_read_flags(target, ioff, isize, flags);
676}
677
678xfs_buf_t *
Nathan Scottce8e9222006-01-11 15:39:08 +1100679xfs_buf_get_empty(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700680 size_t len,
681 xfs_buftarg_t *target)
682{
Nathan Scottce8e9222006-01-11 15:39:08 +1100683 xfs_buf_t *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700684
Nathan Scottce8e9222006-01-11 15:39:08 +1100685 bp = xfs_buf_allocate(0);
686 if (bp)
687 _xfs_buf_initialize(bp, target, 0, len, 0);
688 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700689}
690
691static inline struct page *
692mem_to_page(
693 void *addr)
694{
695 if (((unsigned long)addr < VMALLOC_START) ||
696 ((unsigned long)addr >= VMALLOC_END)) {
697 return virt_to_page(addr);
698 } else {
699 return vmalloc_to_page(addr);
700 }
701}
702
703int
Nathan Scottce8e9222006-01-11 15:39:08 +1100704xfs_buf_associate_memory(
705 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706 void *mem,
707 size_t len)
708{
709 int rval;
710 int i = 0;
711 size_t ptr;
712 size_t end, end_cur;
713 off_t offset;
714 int page_count;
715
716 page_count = PAGE_CACHE_ALIGN(len) >> PAGE_CACHE_SHIFT;
717 offset = (off_t) mem - ((off_t)mem & PAGE_CACHE_MASK);
718 if (offset && (len > PAGE_CACHE_SIZE))
719 page_count++;
720
721 /* Free any previous set of page pointers */
Nathan Scottce8e9222006-01-11 15:39:08 +1100722 if (bp->b_pages)
723 _xfs_buf_free_pages(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700724
Nathan Scottce8e9222006-01-11 15:39:08 +1100725 bp->b_pages = NULL;
726 bp->b_addr = mem;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700727
Nathan Scottce8e9222006-01-11 15:39:08 +1100728 rval = _xfs_buf_get_pages(bp, page_count, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700729 if (rval)
730 return rval;
731
Nathan Scottce8e9222006-01-11 15:39:08 +1100732 bp->b_offset = offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700733 ptr = (size_t) mem & PAGE_CACHE_MASK;
734 end = PAGE_CACHE_ALIGN((size_t) mem + len);
735 end_cur = end;
736 /* set up first page */
Nathan Scottce8e9222006-01-11 15:39:08 +1100737 bp->b_pages[0] = mem_to_page(mem);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700738
739 ptr += PAGE_CACHE_SIZE;
Nathan Scottce8e9222006-01-11 15:39:08 +1100740 bp->b_page_count = ++i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700741 while (ptr < end) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100742 bp->b_pages[i] = mem_to_page((void *)ptr);
743 bp->b_page_count = ++i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700744 ptr += PAGE_CACHE_SIZE;
745 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100746 bp->b_locked = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700747
Nathan Scottce8e9222006-01-11 15:39:08 +1100748 bp->b_count_desired = bp->b_buffer_length = len;
749 bp->b_flags |= XBF_MAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700750
751 return 0;
752}
753
754xfs_buf_t *
Nathan Scottce8e9222006-01-11 15:39:08 +1100755xfs_buf_get_noaddr(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756 size_t len,
757 xfs_buftarg_t *target)
758{
759 size_t malloc_len = len;
760 xfs_buf_t *bp;
761 void *data;
762 int error;
763
Nathan Scottce8e9222006-01-11 15:39:08 +1100764 bp = xfs_buf_allocate(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700765 if (unlikely(bp == NULL))
766 goto fail;
Nathan Scottce8e9222006-01-11 15:39:08 +1100767 _xfs_buf_initialize(bp, target, 0, len, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700768
769 try_again:
770 data = kmem_alloc(malloc_len, KM_SLEEP | KM_MAYFAIL);
771 if (unlikely(data == NULL))
772 goto fail_free_buf;
773
774 /* check whether alignment matches.. */
775 if ((__psunsigned_t)data !=
Nathan Scottce8e9222006-01-11 15:39:08 +1100776 ((__psunsigned_t)data & ~target->bt_smask)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700777 /* .. else double the size and try again */
778 kmem_free(data, malloc_len);
779 malloc_len <<= 1;
780 goto try_again;
781 }
782
Nathan Scottce8e9222006-01-11 15:39:08 +1100783 error = xfs_buf_associate_memory(bp, data, len);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700784 if (error)
785 goto fail_free_mem;
Nathan Scottce8e9222006-01-11 15:39:08 +1100786 bp->b_flags |= _XBF_KMEM_ALLOC;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700787
Nathan Scottce8e9222006-01-11 15:39:08 +1100788 xfs_buf_unlock(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700789
Nathan Scottce8e9222006-01-11 15:39:08 +1100790 XB_TRACE(bp, "no_daddr", data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700791 return bp;
792 fail_free_mem:
793 kmem_free(data, malloc_len);
794 fail_free_buf:
Nathan Scottce8e9222006-01-11 15:39:08 +1100795 xfs_buf_free(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700796 fail:
797 return NULL;
798}
799
800/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700801 * Increment reference count on buffer, to hold the buffer concurrently
802 * with another thread which may release (free) the buffer asynchronously.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700803 * Must hold the buffer already to call this function.
804 */
805void
Nathan Scottce8e9222006-01-11 15:39:08 +1100806xfs_buf_hold(
807 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700808{
Nathan Scottce8e9222006-01-11 15:39:08 +1100809 atomic_inc(&bp->b_hold);
810 XB_TRACE(bp, "hold", 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700811}
812
813/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100814 * Releases a hold on the specified buffer. If the
815 * the hold count is 1, calls xfs_buf_free.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700816 */
817void
Nathan Scottce8e9222006-01-11 15:39:08 +1100818xfs_buf_rele(
819 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700820{
Nathan Scottce8e9222006-01-11 15:39:08 +1100821 xfs_bufhash_t *hash = bp->b_hash;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700822
Nathan Scottce8e9222006-01-11 15:39:08 +1100823 XB_TRACE(bp, "rele", bp->b_relse);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700824
Nathan Scottfad3aa12006-02-01 12:14:52 +1100825 if (unlikely(!hash)) {
826 ASSERT(!bp->b_relse);
827 if (atomic_dec_and_test(&bp->b_hold))
828 xfs_buf_free(bp);
829 return;
830 }
831
Nathan Scottce8e9222006-01-11 15:39:08 +1100832 if (atomic_dec_and_lock(&bp->b_hold, &hash->bh_lock)) {
833 if (bp->b_relse) {
834 atomic_inc(&bp->b_hold);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700835 spin_unlock(&hash->bh_lock);
Nathan Scottce8e9222006-01-11 15:39:08 +1100836 (*(bp->b_relse)) (bp);
837 } else if (bp->b_flags & XBF_FS_MANAGED) {
Christoph Hellwig7f14d0a2005-11-02 15:09:35 +1100838 spin_unlock(&hash->bh_lock);
839 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100840 ASSERT(!(bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)));
841 list_del_init(&bp->b_hash_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700842 spin_unlock(&hash->bh_lock);
Nathan Scottce8e9222006-01-11 15:39:08 +1100843 xfs_buf_free(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700844 }
David Chinner2f926582005-09-05 08:33:35 +1000845 } else {
846 /*
847 * Catch reference count leaks
848 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100849 ASSERT(atomic_read(&bp->b_hold) >= 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700850 }
851}
852
853
854/*
855 * Mutual exclusion on buffers. Locking model:
856 *
857 * Buffers associated with inodes for which buffer locking
858 * is not enabled are not protected by semaphores, and are
859 * assumed to be exclusively owned by the caller. There is a
860 * spinlock in the buffer, used by the caller when concurrent
861 * access is possible.
862 */
863
864/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100865 * Locks a buffer object, if it is not already locked.
866 * Note that this in no way locks the underlying pages, so it is only
867 * useful for synchronizing concurrent use of buffer objects, not for
868 * synchronizing independent access to the underlying pages.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700869 */
870int
Nathan Scottce8e9222006-01-11 15:39:08 +1100871xfs_buf_cond_lock(
872 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700873{
874 int locked;
875
Nathan Scottce8e9222006-01-11 15:39:08 +1100876 locked = down_trylock(&bp->b_sema) == 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700877 if (locked) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100878 XB_SET_OWNER(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700879 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100880 XB_TRACE(bp, "cond_lock", (long)locked);
881 return locked ? 0 : -EBUSY;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700882}
883
884#if defined(DEBUG) || defined(XFS_BLI_TRACE)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700885int
Nathan Scottce8e9222006-01-11 15:39:08 +1100886xfs_buf_lock_value(
887 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700888{
Nathan Scottce8e9222006-01-11 15:39:08 +1100889 return atomic_read(&bp->b_sema.count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700890}
891#endif
892
893/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100894 * Locks a buffer object.
895 * Note that this in no way locks the underlying pages, so it is only
896 * useful for synchronizing concurrent use of buffer objects, not for
897 * synchronizing independent access to the underlying pages.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700898 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100899void
900xfs_buf_lock(
901 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700902{
Nathan Scottce8e9222006-01-11 15:39:08 +1100903 XB_TRACE(bp, "lock", 0);
904 if (atomic_read(&bp->b_io_remaining))
905 blk_run_address_space(bp->b_target->bt_mapping);
906 down(&bp->b_sema);
907 XB_SET_OWNER(bp);
908 XB_TRACE(bp, "locked", 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700909}
910
911/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100912 * Releases the lock on the buffer object.
David Chinner2f926582005-09-05 08:33:35 +1000913 * If the buffer is marked delwri but is not queued, do so before we
Nathan Scottce8e9222006-01-11 15:39:08 +1100914 * unlock the buffer as we need to set flags correctly. We also need to
David Chinner2f926582005-09-05 08:33:35 +1000915 * take a reference for the delwri queue because the unlocker is going to
916 * drop their's and they don't know we just queued it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700917 */
918void
Nathan Scottce8e9222006-01-11 15:39:08 +1100919xfs_buf_unlock(
920 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700921{
Nathan Scottce8e9222006-01-11 15:39:08 +1100922 if ((bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)) == XBF_DELWRI) {
923 atomic_inc(&bp->b_hold);
924 bp->b_flags |= XBF_ASYNC;
925 xfs_buf_delwri_queue(bp, 0);
David Chinner2f926582005-09-05 08:33:35 +1000926 }
927
Nathan Scottce8e9222006-01-11 15:39:08 +1100928 XB_CLEAR_OWNER(bp);
929 up(&bp->b_sema);
930 XB_TRACE(bp, "unlock", 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700931}
932
933
934/*
935 * Pinning Buffer Storage in Memory
Nathan Scottce8e9222006-01-11 15:39:08 +1100936 * Ensure that no attempt to force a buffer to disk will succeed.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700937 */
938void
Nathan Scottce8e9222006-01-11 15:39:08 +1100939xfs_buf_pin(
940 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700941{
Nathan Scottce8e9222006-01-11 15:39:08 +1100942 atomic_inc(&bp->b_pin_count);
943 XB_TRACE(bp, "pin", (long)bp->b_pin_count.counter);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700944}
945
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946void
Nathan Scottce8e9222006-01-11 15:39:08 +1100947xfs_buf_unpin(
948 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700949{
Nathan Scottce8e9222006-01-11 15:39:08 +1100950 if (atomic_dec_and_test(&bp->b_pin_count))
951 wake_up_all(&bp->b_waiters);
952 XB_TRACE(bp, "unpin", (long)bp->b_pin_count.counter);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700953}
954
955int
Nathan Scottce8e9222006-01-11 15:39:08 +1100956xfs_buf_ispin(
957 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958{
Nathan Scottce8e9222006-01-11 15:39:08 +1100959 return atomic_read(&bp->b_pin_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700960}
961
Nathan Scottce8e9222006-01-11 15:39:08 +1100962STATIC void
963xfs_buf_wait_unpin(
964 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700965{
966 DECLARE_WAITQUEUE (wait, current);
967
Nathan Scottce8e9222006-01-11 15:39:08 +1100968 if (atomic_read(&bp->b_pin_count) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969 return;
970
Nathan Scottce8e9222006-01-11 15:39:08 +1100971 add_wait_queue(&bp->b_waiters, &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700972 for (;;) {
973 set_current_state(TASK_UNINTERRUPTIBLE);
Nathan Scottce8e9222006-01-11 15:39:08 +1100974 if (atomic_read(&bp->b_pin_count) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700975 break;
Nathan Scottce8e9222006-01-11 15:39:08 +1100976 if (atomic_read(&bp->b_io_remaining))
977 blk_run_address_space(bp->b_target->bt_mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700978 schedule();
979 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100980 remove_wait_queue(&bp->b_waiters, &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700981 set_current_state(TASK_RUNNING);
982}
983
984/*
985 * Buffer Utility Routines
986 */
987
Linus Torvalds1da177e2005-04-16 15:20:36 -0700988STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +1100989xfs_buf_iodone_work(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700990 void *v)
991{
992 xfs_buf_t *bp = (xfs_buf_t *)v;
993
Nathan Scottce8e9222006-01-11 15:39:08 +1100994 if (bp->b_iodone)
995 (*(bp->b_iodone))(bp);
996 else if (bp->b_flags & XBF_ASYNC)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700997 xfs_buf_relse(bp);
998}
999
1000void
Nathan Scottce8e9222006-01-11 15:39:08 +11001001xfs_buf_ioend(
1002 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001003 int schedule)
1004{
Nathan Scottce8e9222006-01-11 15:39:08 +11001005 bp->b_flags &= ~(XBF_READ | XBF_WRITE);
1006 if (bp->b_error == 0)
1007 bp->b_flags |= XBF_DONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001008
Nathan Scottce8e9222006-01-11 15:39:08 +11001009 XB_TRACE(bp, "iodone", bp->b_iodone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001010
Nathan Scottce8e9222006-01-11 15:39:08 +11001011 if ((bp->b_iodone) || (bp->b_flags & XBF_ASYNC)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001012 if (schedule) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001013 INIT_WORK(&bp->b_iodone_work, xfs_buf_iodone_work, bp);
1014 queue_work(xfslogd_workqueue, &bp->b_iodone_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001015 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +11001016 xfs_buf_iodone_work(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001017 }
1018 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +11001019 up(&bp->b_iodonesema);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001020 }
1021}
1022
Linus Torvalds1da177e2005-04-16 15:20:36 -07001023void
Nathan Scottce8e9222006-01-11 15:39:08 +11001024xfs_buf_ioerror(
1025 xfs_buf_t *bp,
1026 int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001027{
1028 ASSERT(error >= 0 && error <= 0xffff);
Nathan Scottce8e9222006-01-11 15:39:08 +11001029 bp->b_error = (unsigned short)error;
1030 XB_TRACE(bp, "ioerror", (unsigned long)error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031}
1032
1033/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001034 * Initiate I/O on a buffer, based on the flags supplied.
1035 * The b_iodone routine in the buffer supplied will only be called
Linus Torvalds1da177e2005-04-16 15:20:36 -07001036 * when all of the subsidiary I/O requests, if any, have been completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001037 */
1038int
Nathan Scottce8e9222006-01-11 15:39:08 +11001039xfs_buf_iostart(
1040 xfs_buf_t *bp,
1041 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001042{
1043 int status = 0;
1044
Nathan Scottce8e9222006-01-11 15:39:08 +11001045 XB_TRACE(bp, "iostart", (unsigned long)flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001046
Nathan Scottce8e9222006-01-11 15:39:08 +11001047 if (flags & XBF_DELWRI) {
1048 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_ASYNC);
1049 bp->b_flags |= flags & (XBF_DELWRI | XBF_ASYNC);
1050 xfs_buf_delwri_queue(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001051 return status;
1052 }
1053
Nathan Scottce8e9222006-01-11 15:39:08 +11001054 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_ASYNC | XBF_DELWRI | \
1055 XBF_READ_AHEAD | _XBF_RUN_QUEUES);
1056 bp->b_flags |= flags & (XBF_READ | XBF_WRITE | XBF_ASYNC | \
1057 XBF_READ_AHEAD | _XBF_RUN_QUEUES);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001058
Nathan Scottce8e9222006-01-11 15:39:08 +11001059 BUG_ON(bp->b_bn == XFS_BUF_DADDR_NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001060
1061 /* For writes allow an alternate strategy routine to precede
1062 * the actual I/O request (which may not be issued at all in
1063 * a shutdown situation, for example).
1064 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001065 status = (flags & XBF_WRITE) ?
1066 xfs_buf_iostrategy(bp) : xfs_buf_iorequest(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001067
1068 /* Wait for I/O if we are not an async request.
1069 * Note: async I/O request completion will release the buffer,
1070 * and that can already be done by this point. So using the
1071 * buffer pointer from here on, after async I/O, is invalid.
1072 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001073 if (!status && !(flags & XBF_ASYNC))
1074 status = xfs_buf_iowait(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001075
1076 return status;
1077}
1078
Linus Torvalds1da177e2005-04-16 15:20:36 -07001079STATIC __inline__ int
Nathan Scottce8e9222006-01-11 15:39:08 +11001080_xfs_buf_iolocked(
1081 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001082{
Nathan Scottce8e9222006-01-11 15:39:08 +11001083 ASSERT(bp->b_flags & (XBF_READ | XBF_WRITE));
1084 if (bp->b_flags & XBF_READ)
1085 return bp->b_locked;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001086 return 0;
1087}
1088
1089STATIC __inline__ void
Nathan Scottce8e9222006-01-11 15:39:08 +11001090_xfs_buf_ioend(
1091 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001092 int schedule)
1093{
Nathan Scottce8e9222006-01-11 15:39:08 +11001094 if (atomic_dec_and_test(&bp->b_io_remaining) == 1) {
1095 bp->b_locked = 0;
1096 xfs_buf_ioend(bp, schedule);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001097 }
1098}
1099
1100STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +11001101xfs_buf_bio_end_io(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001102 struct bio *bio,
1103 unsigned int bytes_done,
1104 int error)
1105{
Nathan Scottce8e9222006-01-11 15:39:08 +11001106 xfs_buf_t *bp = (xfs_buf_t *)bio->bi_private;
1107 unsigned int blocksize = bp->b_target->bt_bsize;
Nathan Scotteedb5532005-09-02 16:39:56 +10001108 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001109
1110 if (bio->bi_size)
1111 return 1;
1112
1113 if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
Nathan Scottce8e9222006-01-11 15:39:08 +11001114 bp->b_error = EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001115
Nathan Scotteedb5532005-09-02 16:39:56 +10001116 do {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001117 struct page *page = bvec->bv_page;
1118
Nathan Scottce8e9222006-01-11 15:39:08 +11001119 if (unlikely(bp->b_error)) {
1120 if (bp->b_flags & XBF_READ)
Nathan Scotteedb5532005-09-02 16:39:56 +10001121 ClearPageUptodate(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001122 SetPageError(page);
Nathan Scottce8e9222006-01-11 15:39:08 +11001123 } else if (blocksize >= PAGE_CACHE_SIZE) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001124 SetPageUptodate(page);
1125 } else if (!PagePrivate(page) &&
Nathan Scottce8e9222006-01-11 15:39:08 +11001126 (bp->b_flags & _XBF_PAGE_CACHE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001127 set_page_region(page, bvec->bv_offset, bvec->bv_len);
1128 }
1129
Nathan Scotteedb5532005-09-02 16:39:56 +10001130 if (--bvec >= bio->bi_io_vec)
1131 prefetchw(&bvec->bv_page->flags);
1132
Nathan Scottce8e9222006-01-11 15:39:08 +11001133 if (_xfs_buf_iolocked(bp)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001134 unlock_page(page);
1135 }
Nathan Scotteedb5532005-09-02 16:39:56 +10001136 } while (bvec >= bio->bi_io_vec);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001137
Nathan Scottce8e9222006-01-11 15:39:08 +11001138 _xfs_buf_ioend(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001139 bio_put(bio);
1140 return 0;
1141}
1142
1143STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001144_xfs_buf_ioapply(
1145 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001146{
1147 int i, rw, map_i, total_nr_pages, nr_pages;
1148 struct bio *bio;
Nathan Scottce8e9222006-01-11 15:39:08 +11001149 int offset = bp->b_offset;
1150 int size = bp->b_count_desired;
1151 sector_t sector = bp->b_bn;
1152 unsigned int blocksize = bp->b_target->bt_bsize;
1153 int locking = _xfs_buf_iolocked(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001154
Nathan Scottce8e9222006-01-11 15:39:08 +11001155 total_nr_pages = bp->b_page_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001156 map_i = 0;
1157
Nathan Scottce8e9222006-01-11 15:39:08 +11001158 if (bp->b_flags & _XBF_RUN_QUEUES) {
1159 bp->b_flags &= ~_XBF_RUN_QUEUES;
1160 rw = (bp->b_flags & XBF_READ) ? READ_SYNC : WRITE_SYNC;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001161 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +11001162 rw = (bp->b_flags & XBF_READ) ? READ : WRITE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001163 }
1164
Nathan Scottce8e9222006-01-11 15:39:08 +11001165 if (bp->b_flags & XBF_ORDERED) {
1166 ASSERT(!(bp->b_flags & XBF_READ));
Christoph Hellwigf538d4d2005-11-02 10:26:59 +11001167 rw = WRITE_BARRIER;
1168 }
1169
Nathan Scottce8e9222006-01-11 15:39:08 +11001170 /* Special code path for reading a sub page size buffer in --
Linus Torvalds1da177e2005-04-16 15:20:36 -07001171 * we populate up the whole page, and hence the other metadata
1172 * in the same page. This optimization is only valid when the
Nathan Scottce8e9222006-01-11 15:39:08 +11001173 * filesystem block size is not smaller than the page size.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001174 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001175 if ((bp->b_buffer_length < PAGE_CACHE_SIZE) &&
1176 (bp->b_flags & XBF_READ) && locking &&
1177 (blocksize >= PAGE_CACHE_SIZE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001178 bio = bio_alloc(GFP_NOIO, 1);
1179
Nathan Scottce8e9222006-01-11 15:39:08 +11001180 bio->bi_bdev = bp->b_target->bt_bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001181 bio->bi_sector = sector - (offset >> BBSHIFT);
Nathan Scottce8e9222006-01-11 15:39:08 +11001182 bio->bi_end_io = xfs_buf_bio_end_io;
1183 bio->bi_private = bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001184
Nathan Scottce8e9222006-01-11 15:39:08 +11001185 bio_add_page(bio, bp->b_pages[0], PAGE_CACHE_SIZE, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001186 size = 0;
1187
Nathan Scottce8e9222006-01-11 15:39:08 +11001188 atomic_inc(&bp->b_io_remaining);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001189
1190 goto submit_io;
1191 }
1192
1193 /* Lock down the pages which we need to for the request */
Nathan Scottce8e9222006-01-11 15:39:08 +11001194 if (locking && (bp->b_flags & XBF_WRITE) && (bp->b_locked == 0)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001195 for (i = 0; size; i++) {
1196 int nbytes = PAGE_CACHE_SIZE - offset;
Nathan Scottce8e9222006-01-11 15:39:08 +11001197 struct page *page = bp->b_pages[i];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001198
1199 if (nbytes > size)
1200 nbytes = size;
1201
1202 lock_page(page);
1203
1204 size -= nbytes;
1205 offset = 0;
1206 }
Nathan Scottce8e9222006-01-11 15:39:08 +11001207 offset = bp->b_offset;
1208 size = bp->b_count_desired;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001209 }
1210
1211next_chunk:
Nathan Scottce8e9222006-01-11 15:39:08 +11001212 atomic_inc(&bp->b_io_remaining);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001213 nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1214 if (nr_pages > total_nr_pages)
1215 nr_pages = total_nr_pages;
1216
1217 bio = bio_alloc(GFP_NOIO, nr_pages);
Nathan Scottce8e9222006-01-11 15:39:08 +11001218 bio->bi_bdev = bp->b_target->bt_bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001219 bio->bi_sector = sector;
Nathan Scottce8e9222006-01-11 15:39:08 +11001220 bio->bi_end_io = xfs_buf_bio_end_io;
1221 bio->bi_private = bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001222
1223 for (; size && nr_pages; nr_pages--, map_i++) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001224 int rbytes, nbytes = PAGE_CACHE_SIZE - offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001225
1226 if (nbytes > size)
1227 nbytes = size;
1228
Nathan Scottce8e9222006-01-11 15:39:08 +11001229 rbytes = bio_add_page(bio, bp->b_pages[map_i], nbytes, offset);
1230 if (rbytes < nbytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001231 break;
1232
1233 offset = 0;
1234 sector += nbytes >> BBSHIFT;
1235 size -= nbytes;
1236 total_nr_pages--;
1237 }
1238
1239submit_io:
1240 if (likely(bio->bi_size)) {
1241 submit_bio(rw, bio);
1242 if (size)
1243 goto next_chunk;
1244 } else {
1245 bio_put(bio);
Nathan Scottce8e9222006-01-11 15:39:08 +11001246 xfs_buf_ioerror(bp, EIO);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001247 }
1248}
1249
Linus Torvalds1da177e2005-04-16 15:20:36 -07001250int
Nathan Scottce8e9222006-01-11 15:39:08 +11001251xfs_buf_iorequest(
1252 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001253{
Nathan Scottce8e9222006-01-11 15:39:08 +11001254 XB_TRACE(bp, "iorequest", 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001255
Nathan Scottce8e9222006-01-11 15:39:08 +11001256 if (bp->b_flags & XBF_DELWRI) {
1257 xfs_buf_delwri_queue(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001258 return 0;
1259 }
1260
Nathan Scottce8e9222006-01-11 15:39:08 +11001261 if (bp->b_flags & XBF_WRITE) {
1262 xfs_buf_wait_unpin(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001263 }
1264
Nathan Scottce8e9222006-01-11 15:39:08 +11001265 xfs_buf_hold(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001266
1267 /* Set the count to 1 initially, this will stop an I/O
1268 * completion callout which happens before we have started
Nathan Scottce8e9222006-01-11 15:39:08 +11001269 * all the I/O from calling xfs_buf_ioend too early.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001270 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001271 atomic_set(&bp->b_io_remaining, 1);
1272 _xfs_buf_ioapply(bp);
1273 _xfs_buf_ioend(bp, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001274
Nathan Scottce8e9222006-01-11 15:39:08 +11001275 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001276 return 0;
1277}
1278
1279/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001280 * Waits for I/O to complete on the buffer supplied.
1281 * It returns immediately if no I/O is pending.
1282 * It returns the I/O error code, if any, or 0 if there was no error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001283 */
1284int
Nathan Scottce8e9222006-01-11 15:39:08 +11001285xfs_buf_iowait(
1286 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001287{
Nathan Scottce8e9222006-01-11 15:39:08 +11001288 XB_TRACE(bp, "iowait", 0);
1289 if (atomic_read(&bp->b_io_remaining))
1290 blk_run_address_space(bp->b_target->bt_mapping);
1291 down(&bp->b_iodonesema);
1292 XB_TRACE(bp, "iowaited", (long)bp->b_error);
1293 return bp->b_error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001294}
1295
Nathan Scottce8e9222006-01-11 15:39:08 +11001296xfs_caddr_t
1297xfs_buf_offset(
1298 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001299 size_t offset)
1300{
1301 struct page *page;
1302
Nathan Scottce8e9222006-01-11 15:39:08 +11001303 if (bp->b_flags & XBF_MAPPED)
1304 return XFS_BUF_PTR(bp) + offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001305
Nathan Scottce8e9222006-01-11 15:39:08 +11001306 offset += bp->b_offset;
1307 page = bp->b_pages[offset >> PAGE_CACHE_SHIFT];
1308 return (xfs_caddr_t)page_address(page) + (offset & (PAGE_CACHE_SIZE-1));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001309}
1310
1311/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001312 * Move data into or out of a buffer.
1313 */
1314void
Nathan Scottce8e9222006-01-11 15:39:08 +11001315xfs_buf_iomove(
1316 xfs_buf_t *bp, /* buffer to process */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001317 size_t boff, /* starting buffer offset */
1318 size_t bsize, /* length to copy */
1319 caddr_t data, /* data address */
Nathan Scottce8e9222006-01-11 15:39:08 +11001320 xfs_buf_rw_t mode) /* read/write/zero flag */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001321{
1322 size_t bend, cpoff, csize;
1323 struct page *page;
1324
1325 bend = boff + bsize;
1326 while (boff < bend) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001327 page = bp->b_pages[xfs_buf_btoct(boff + bp->b_offset)];
1328 cpoff = xfs_buf_poff(boff + bp->b_offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001329 csize = min_t(size_t,
Nathan Scottce8e9222006-01-11 15:39:08 +11001330 PAGE_CACHE_SIZE-cpoff, bp->b_count_desired-boff);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001331
1332 ASSERT(((csize + cpoff) <= PAGE_CACHE_SIZE));
1333
1334 switch (mode) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001335 case XBRW_ZERO:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001336 memset(page_address(page) + cpoff, 0, csize);
1337 break;
Nathan Scottce8e9222006-01-11 15:39:08 +11001338 case XBRW_READ:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001339 memcpy(data, page_address(page) + cpoff, csize);
1340 break;
Nathan Scottce8e9222006-01-11 15:39:08 +11001341 case XBRW_WRITE:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001342 memcpy(page_address(page) + cpoff, data, csize);
1343 }
1344
1345 boff += csize;
1346 data += csize;
1347 }
1348}
1349
1350/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001351 * Handling of buffer targets (buftargs).
Linus Torvalds1da177e2005-04-16 15:20:36 -07001352 */
1353
1354/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001355 * Wait for any bufs with callbacks that have been submitted but
1356 * have not yet returned... walk the hash list for the target.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001357 */
1358void
1359xfs_wait_buftarg(
1360 xfs_buftarg_t *btp)
1361{
1362 xfs_buf_t *bp, *n;
1363 xfs_bufhash_t *hash;
1364 uint i;
1365
1366 for (i = 0; i < (1 << btp->bt_hashshift); i++) {
1367 hash = &btp->bt_hash[i];
1368again:
1369 spin_lock(&hash->bh_lock);
Nathan Scottce8e9222006-01-11 15:39:08 +11001370 list_for_each_entry_safe(bp, n, &hash->bh_list, b_hash_list) {
1371 ASSERT(btp == bp->b_target);
1372 if (!(bp->b_flags & XBF_FS_MANAGED)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001373 spin_unlock(&hash->bh_lock);
David Chinner2f926582005-09-05 08:33:35 +10001374 /*
1375 * Catch superblock reference count leaks
1376 * immediately
1377 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001378 BUG_ON(bp->b_bn == 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001379 delay(100);
1380 goto again;
1381 }
1382 }
1383 spin_unlock(&hash->bh_lock);
1384 }
1385}
1386
1387/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001388 * Allocate buffer hash table for a given target.
1389 * For devices containing metadata (i.e. not the log/realtime devices)
1390 * we need to allocate a much larger hash table.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001391 */
1392STATIC void
1393xfs_alloc_bufhash(
1394 xfs_buftarg_t *btp,
1395 int external)
1396{
1397 unsigned int i;
1398
1399 btp->bt_hashshift = external ? 3 : 8; /* 8 or 256 buckets */
1400 btp->bt_hashmask = (1 << btp->bt_hashshift) - 1;
1401 btp->bt_hash = kmem_zalloc((1 << btp->bt_hashshift) *
1402 sizeof(xfs_bufhash_t), KM_SLEEP);
1403 for (i = 0; i < (1 << btp->bt_hashshift); i++) {
1404 spin_lock_init(&btp->bt_hash[i].bh_lock);
1405 INIT_LIST_HEAD(&btp->bt_hash[i].bh_list);
1406 }
1407}
1408
1409STATIC void
1410xfs_free_bufhash(
1411 xfs_buftarg_t *btp)
1412{
Nathan Scottce8e9222006-01-11 15:39:08 +11001413 kmem_free(btp->bt_hash, (1<<btp->bt_hashshift) * sizeof(xfs_bufhash_t));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001414 btp->bt_hash = NULL;
1415}
1416
David Chinnera6867a62006-01-11 15:37:58 +11001417/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001418 * buftarg list for delwrite queue processing
David Chinnera6867a62006-01-11 15:37:58 +11001419 */
1420STATIC LIST_HEAD(xfs_buftarg_list);
1421STATIC DEFINE_SPINLOCK(xfs_buftarg_lock);
1422
1423STATIC void
1424xfs_register_buftarg(
1425 xfs_buftarg_t *btp)
1426{
1427 spin_lock(&xfs_buftarg_lock);
1428 list_add(&btp->bt_list, &xfs_buftarg_list);
1429 spin_unlock(&xfs_buftarg_lock);
1430}
1431
1432STATIC void
1433xfs_unregister_buftarg(
1434 xfs_buftarg_t *btp)
1435{
1436 spin_lock(&xfs_buftarg_lock);
1437 list_del(&btp->bt_list);
1438 spin_unlock(&xfs_buftarg_lock);
1439}
1440
Linus Torvalds1da177e2005-04-16 15:20:36 -07001441void
1442xfs_free_buftarg(
1443 xfs_buftarg_t *btp,
1444 int external)
1445{
1446 xfs_flush_buftarg(btp, 1);
1447 if (external)
Nathan Scottce8e9222006-01-11 15:39:08 +11001448 xfs_blkdev_put(btp->bt_bdev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001449 xfs_free_bufhash(btp);
Nathan Scottce8e9222006-01-11 15:39:08 +11001450 iput(btp->bt_mapping->host);
David Chinnera6867a62006-01-11 15:37:58 +11001451
Nathan Scottce8e9222006-01-11 15:39:08 +11001452 /* Unregister the buftarg first so that we don't get a
1453 * wakeup finding a non-existent task
1454 */
David Chinnera6867a62006-01-11 15:37:58 +11001455 xfs_unregister_buftarg(btp);
1456 kthread_stop(btp->bt_task);
1457
Linus Torvalds1da177e2005-04-16 15:20:36 -07001458 kmem_free(btp, sizeof(*btp));
1459}
1460
Linus Torvalds1da177e2005-04-16 15:20:36 -07001461STATIC int
1462xfs_setsize_buftarg_flags(
1463 xfs_buftarg_t *btp,
1464 unsigned int blocksize,
1465 unsigned int sectorsize,
1466 int verbose)
1467{
Nathan Scottce8e9222006-01-11 15:39:08 +11001468 btp->bt_bsize = blocksize;
1469 btp->bt_sshift = ffs(sectorsize) - 1;
1470 btp->bt_smask = sectorsize - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001471
Nathan Scottce8e9222006-01-11 15:39:08 +11001472 if (set_blocksize(btp->bt_bdev, sectorsize)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001473 printk(KERN_WARNING
1474 "XFS: Cannot set_blocksize to %u on device %s\n",
1475 sectorsize, XFS_BUFTARG_NAME(btp));
1476 return EINVAL;
1477 }
1478
1479 if (verbose &&
1480 (PAGE_CACHE_SIZE / BITS_PER_LONG) > sectorsize) {
1481 printk(KERN_WARNING
1482 "XFS: %u byte sectors in use on device %s. "
1483 "This is suboptimal; %u or greater is ideal.\n",
1484 sectorsize, XFS_BUFTARG_NAME(btp),
1485 (unsigned int)PAGE_CACHE_SIZE / BITS_PER_LONG);
1486 }
1487
1488 return 0;
1489}
1490
1491/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001492 * When allocating the initial buffer target we have not yet
1493 * read in the superblock, so don't know what sized sectors
1494 * are being used is at this early stage. Play safe.
1495 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001496STATIC int
1497xfs_setsize_buftarg_early(
1498 xfs_buftarg_t *btp,
1499 struct block_device *bdev)
1500{
1501 return xfs_setsize_buftarg_flags(btp,
1502 PAGE_CACHE_SIZE, bdev_hardsect_size(bdev), 0);
1503}
1504
1505int
1506xfs_setsize_buftarg(
1507 xfs_buftarg_t *btp,
1508 unsigned int blocksize,
1509 unsigned int sectorsize)
1510{
1511 return xfs_setsize_buftarg_flags(btp, blocksize, sectorsize, 1);
1512}
1513
1514STATIC int
1515xfs_mapping_buftarg(
1516 xfs_buftarg_t *btp,
1517 struct block_device *bdev)
1518{
1519 struct backing_dev_info *bdi;
1520 struct inode *inode;
1521 struct address_space *mapping;
1522 static struct address_space_operations mapping_aops = {
1523 .sync_page = block_sync_page,
1524 };
1525
1526 inode = new_inode(bdev->bd_inode->i_sb);
1527 if (!inode) {
1528 printk(KERN_WARNING
1529 "XFS: Cannot allocate mapping inode for device %s\n",
1530 XFS_BUFTARG_NAME(btp));
1531 return ENOMEM;
1532 }
1533 inode->i_mode = S_IFBLK;
1534 inode->i_bdev = bdev;
1535 inode->i_rdev = bdev->bd_dev;
1536 bdi = blk_get_backing_dev_info(bdev);
1537 if (!bdi)
1538 bdi = &default_backing_dev_info;
1539 mapping = &inode->i_data;
1540 mapping->a_ops = &mapping_aops;
1541 mapping->backing_dev_info = bdi;
1542 mapping_set_gfp_mask(mapping, GFP_NOFS);
Nathan Scottce8e9222006-01-11 15:39:08 +11001543 btp->bt_mapping = mapping;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001544 return 0;
1545}
1546
David Chinnera6867a62006-01-11 15:37:58 +11001547STATIC int
1548xfs_alloc_delwrite_queue(
1549 xfs_buftarg_t *btp)
1550{
1551 int error = 0;
1552
1553 INIT_LIST_HEAD(&btp->bt_list);
1554 INIT_LIST_HEAD(&btp->bt_delwrite_queue);
1555 spinlock_init(&btp->bt_delwrite_lock, "delwri_lock");
1556 btp->bt_flags = 0;
1557 btp->bt_task = kthread_run(xfsbufd, btp, "xfsbufd");
1558 if (IS_ERR(btp->bt_task)) {
1559 error = PTR_ERR(btp->bt_task);
1560 goto out_error;
1561 }
1562 xfs_register_buftarg(btp);
1563out_error:
1564 return error;
1565}
1566
Linus Torvalds1da177e2005-04-16 15:20:36 -07001567xfs_buftarg_t *
1568xfs_alloc_buftarg(
1569 struct block_device *bdev,
1570 int external)
1571{
1572 xfs_buftarg_t *btp;
1573
1574 btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
1575
Nathan Scottce8e9222006-01-11 15:39:08 +11001576 btp->bt_dev = bdev->bd_dev;
1577 btp->bt_bdev = bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001578 if (xfs_setsize_buftarg_early(btp, bdev))
1579 goto error;
1580 if (xfs_mapping_buftarg(btp, bdev))
1581 goto error;
David Chinnera6867a62006-01-11 15:37:58 +11001582 if (xfs_alloc_delwrite_queue(btp))
1583 goto error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001584 xfs_alloc_bufhash(btp, external);
1585 return btp;
1586
1587error:
1588 kmem_free(btp, sizeof(*btp));
1589 return NULL;
1590}
1591
1592
1593/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001594 * Delayed write buffer handling
Linus Torvalds1da177e2005-04-16 15:20:36 -07001595 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001596STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001597xfs_buf_delwri_queue(
1598 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001599 int unlock)
1600{
Nathan Scottce8e9222006-01-11 15:39:08 +11001601 struct list_head *dwq = &bp->b_target->bt_delwrite_queue;
1602 spinlock_t *dwlk = &bp->b_target->bt_delwrite_lock;
David Chinnera6867a62006-01-11 15:37:58 +11001603
Nathan Scottce8e9222006-01-11 15:39:08 +11001604 XB_TRACE(bp, "delwri_q", (long)unlock);
1605 ASSERT((bp->b_flags&(XBF_DELWRI|XBF_ASYNC)) == (XBF_DELWRI|XBF_ASYNC));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001606
David Chinnera6867a62006-01-11 15:37:58 +11001607 spin_lock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001608 /* If already in the queue, dequeue and place at tail */
Nathan Scottce8e9222006-01-11 15:39:08 +11001609 if (!list_empty(&bp->b_list)) {
1610 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1611 if (unlock)
1612 atomic_dec(&bp->b_hold);
1613 list_del(&bp->b_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001614 }
1615
Nathan Scottce8e9222006-01-11 15:39:08 +11001616 bp->b_flags |= _XBF_DELWRI_Q;
1617 list_add_tail(&bp->b_list, dwq);
1618 bp->b_queuetime = jiffies;
David Chinnera6867a62006-01-11 15:37:58 +11001619 spin_unlock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001620
1621 if (unlock)
Nathan Scottce8e9222006-01-11 15:39:08 +11001622 xfs_buf_unlock(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001623}
1624
1625void
Nathan Scottce8e9222006-01-11 15:39:08 +11001626xfs_buf_delwri_dequeue(
1627 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001628{
Nathan Scottce8e9222006-01-11 15:39:08 +11001629 spinlock_t *dwlk = &bp->b_target->bt_delwrite_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001630 int dequeued = 0;
1631
David Chinnera6867a62006-01-11 15:37:58 +11001632 spin_lock(dwlk);
Nathan Scottce8e9222006-01-11 15:39:08 +11001633 if ((bp->b_flags & XBF_DELWRI) && !list_empty(&bp->b_list)) {
1634 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1635 list_del_init(&bp->b_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001636 dequeued = 1;
1637 }
Nathan Scottce8e9222006-01-11 15:39:08 +11001638 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
David Chinnera6867a62006-01-11 15:37:58 +11001639 spin_unlock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001640
1641 if (dequeued)
Nathan Scottce8e9222006-01-11 15:39:08 +11001642 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001643
Nathan Scottce8e9222006-01-11 15:39:08 +11001644 XB_TRACE(bp, "delwri_dq", (long)dequeued);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001645}
1646
1647STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001648xfs_buf_runall_queues(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001649 struct workqueue_struct *queue)
1650{
1651 flush_workqueue(queue);
1652}
1653
Linus Torvalds1da177e2005-04-16 15:20:36 -07001654STATIC int
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001655xfsbufd_wakeup(
Nathan Scott15c84a42005-11-04 10:51:01 +11001656 int priority,
1657 gfp_t mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001658{
Christoph Hellwigda7f93e2006-01-11 20:49:57 +11001659 xfs_buftarg_t *btp;
David Chinnera6867a62006-01-11 15:37:58 +11001660
1661 spin_lock(&xfs_buftarg_lock);
Christoph Hellwigda7f93e2006-01-11 20:49:57 +11001662 list_for_each_entry(btp, &xfs_buftarg_list, bt_list) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001663 if (test_bit(XBT_FORCE_SLEEP, &btp->bt_flags))
David Chinnera6867a62006-01-11 15:37:58 +11001664 continue;
Nathan Scottce8e9222006-01-11 15:39:08 +11001665 set_bit(XBT_FORCE_FLUSH, &btp->bt_flags);
David Chinnera6867a62006-01-11 15:37:58 +11001666 wake_up_process(btp->bt_task);
1667 }
1668 spin_unlock(&xfs_buftarg_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001669 return 0;
1670}
1671
1672STATIC int
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001673xfsbufd(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001674 void *data)
1675{
1676 struct list_head tmp;
1677 unsigned long age;
David Chinnera6867a62006-01-11 15:37:58 +11001678 xfs_buftarg_t *target = (xfs_buftarg_t *)data;
Nathan Scottce8e9222006-01-11 15:39:08 +11001679 xfs_buf_t *bp, *n;
David Chinnera6867a62006-01-11 15:37:58 +11001680 struct list_head *dwq = &target->bt_delwrite_queue;
1681 spinlock_t *dwlk = &target->bt_delwrite_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001682
Linus Torvalds1da177e2005-04-16 15:20:36 -07001683 current->flags |= PF_MEMALLOC;
1684
Linus Torvalds1da177e2005-04-16 15:20:36 -07001685 INIT_LIST_HEAD(&tmp);
1686 do {
Christoph Lameter3e1d1d22005-06-24 23:13:50 -07001687 if (unlikely(freezing(current))) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001688 set_bit(XBT_FORCE_SLEEP, &target->bt_flags);
Christoph Lameter3e1d1d22005-06-24 23:13:50 -07001689 refrigerator();
Nathan Scottabd0cf72005-05-05 13:30:13 -07001690 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +11001691 clear_bit(XBT_FORCE_SLEEP, &target->bt_flags);
Nathan Scottabd0cf72005-05-05 13:30:13 -07001692 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001693
Nathan Scott15c84a42005-11-04 10:51:01 +11001694 schedule_timeout_interruptible(
1695 xfs_buf_timer_centisecs * msecs_to_jiffies(10));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001696
Nishanth Aravamudan041e0e32005-09-10 00:27:23 -07001697 age = xfs_buf_age_centisecs * msecs_to_jiffies(10);
David Chinnera6867a62006-01-11 15:37:58 +11001698 spin_lock(dwlk);
Nathan Scottce8e9222006-01-11 15:39:08 +11001699 list_for_each_entry_safe(bp, n, dwq, b_list) {
1700 XB_TRACE(bp, "walkq1", (long)xfs_buf_ispin(bp));
1701 ASSERT(bp->b_flags & XBF_DELWRI);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001702
Nathan Scottce8e9222006-01-11 15:39:08 +11001703 if (!xfs_buf_ispin(bp) && !xfs_buf_cond_lock(bp)) {
1704 if (!test_bit(XBT_FORCE_FLUSH,
David Chinnera6867a62006-01-11 15:37:58 +11001705 &target->bt_flags) &&
Linus Torvalds1da177e2005-04-16 15:20:36 -07001706 time_before(jiffies,
Nathan Scottce8e9222006-01-11 15:39:08 +11001707 bp->b_queuetime + age)) {
1708 xfs_buf_unlock(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001709 break;
1710 }
1711
Nathan Scottce8e9222006-01-11 15:39:08 +11001712 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
1713 bp->b_flags |= XBF_WRITE;
1714 list_move(&bp->b_list, &tmp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001715 }
1716 }
David Chinnera6867a62006-01-11 15:37:58 +11001717 spin_unlock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001718
1719 while (!list_empty(&tmp)) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001720 bp = list_entry(tmp.next, xfs_buf_t, b_list);
1721 ASSERT(target == bp->b_target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001722
Nathan Scottce8e9222006-01-11 15:39:08 +11001723 list_del_init(&bp->b_list);
1724 xfs_buf_iostrategy(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001725
Nathan Scottce8e9222006-01-11 15:39:08 +11001726 blk_run_address_space(target->bt_mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001727 }
1728
1729 if (as_list_len > 0)
1730 purge_addresses();
1731
Nathan Scottce8e9222006-01-11 15:39:08 +11001732 clear_bit(XBT_FORCE_FLUSH, &target->bt_flags);
Christoph Hellwig4df08c52005-09-05 08:34:18 +10001733 } while (!kthread_should_stop());
Linus Torvalds1da177e2005-04-16 15:20:36 -07001734
Christoph Hellwig4df08c52005-09-05 08:34:18 +10001735 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001736}
1737
1738/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001739 * Go through all incore buffers, and release buffers if they belong to
1740 * the given device. This is used in filesystem error handling to
1741 * preserve the consistency of its metadata.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001742 */
1743int
1744xfs_flush_buftarg(
1745 xfs_buftarg_t *target,
1746 int wait)
1747{
1748 struct list_head tmp;
Nathan Scottce8e9222006-01-11 15:39:08 +11001749 xfs_buf_t *bp, *n;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001750 int pincount = 0;
David Chinnera6867a62006-01-11 15:37:58 +11001751 struct list_head *dwq = &target->bt_delwrite_queue;
1752 spinlock_t *dwlk = &target->bt_delwrite_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001753
Nathan Scottce8e9222006-01-11 15:39:08 +11001754 xfs_buf_runall_queues(xfsdatad_workqueue);
1755 xfs_buf_runall_queues(xfslogd_workqueue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001756
1757 INIT_LIST_HEAD(&tmp);
David Chinnera6867a62006-01-11 15:37:58 +11001758 spin_lock(dwlk);
Nathan Scottce8e9222006-01-11 15:39:08 +11001759 list_for_each_entry_safe(bp, n, dwq, b_list) {
1760 ASSERT(bp->b_target == target);
1761 ASSERT(bp->b_flags & (XBF_DELWRI | _XBF_DELWRI_Q));
1762 XB_TRACE(bp, "walkq2", (long)xfs_buf_ispin(bp));
1763 if (xfs_buf_ispin(bp)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001764 pincount++;
1765 continue;
1766 }
1767
Nathan Scottce8e9222006-01-11 15:39:08 +11001768 list_move(&bp->b_list, &tmp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001769 }
David Chinnera6867a62006-01-11 15:37:58 +11001770 spin_unlock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001771
1772 /*
1773 * Dropped the delayed write list lock, now walk the temporary list
1774 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001775 list_for_each_entry_safe(bp, n, &tmp, b_list) {
1776 xfs_buf_lock(bp);
1777 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
1778 bp->b_flags |= XBF_WRITE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001779 if (wait)
Nathan Scottce8e9222006-01-11 15:39:08 +11001780 bp->b_flags &= ~XBF_ASYNC;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001781 else
Nathan Scottce8e9222006-01-11 15:39:08 +11001782 list_del_init(&bp->b_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001783
Nathan Scottce8e9222006-01-11 15:39:08 +11001784 xfs_buf_iostrategy(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001785 }
1786
1787 /*
1788 * Remaining list items must be flushed before returning
1789 */
1790 while (!list_empty(&tmp)) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001791 bp = list_entry(tmp.next, xfs_buf_t, b_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001792
Nathan Scottce8e9222006-01-11 15:39:08 +11001793 list_del_init(&bp->b_list);
1794 xfs_iowait(bp);
1795 xfs_buf_relse(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001796 }
1797
1798 if (wait)
Nathan Scottce8e9222006-01-11 15:39:08 +11001799 blk_run_address_space(target->bt_mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001800
1801 return pincount;
1802}
1803
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001804int __init
Nathan Scottce8e9222006-01-11 15:39:08 +11001805xfs_buf_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001806{
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001807 int error = -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001808
Nathan Scottce8e9222006-01-11 15:39:08 +11001809#ifdef XFS_BUF_TRACE
1810 xfs_buf_trace_buf = ktrace_alloc(XFS_BUF_TRACE_SIZE, KM_SLEEP);
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001811#endif
1812
Nathan Scottce8e9222006-01-11 15:39:08 +11001813 xfs_buf_zone = kmem_zone_init(sizeof(xfs_buf_t), "xfs_buf");
1814 if (!xfs_buf_zone)
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001815 goto out_free_trace_buf;
1816
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001817 xfslogd_workqueue = create_workqueue("xfslogd");
1818 if (!xfslogd_workqueue)
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001819 goto out_free_buf_zone;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001820
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001821 xfsdatad_workqueue = create_workqueue("xfsdatad");
1822 if (!xfsdatad_workqueue)
1823 goto out_destroy_xfslogd_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001824
Nathan Scottce8e9222006-01-11 15:39:08 +11001825 xfs_buf_shake = kmem_shake_register(xfsbufd_wakeup);
1826 if (!xfs_buf_shake)
David Chinnera6867a62006-01-11 15:37:58 +11001827 goto out_destroy_xfsdatad_workqueue;
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001828
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001829 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001830
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001831 out_destroy_xfsdatad_workqueue:
1832 destroy_workqueue(xfsdatad_workqueue);
1833 out_destroy_xfslogd_workqueue:
1834 destroy_workqueue(xfslogd_workqueue);
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001835 out_free_buf_zone:
Nathan Scottce8e9222006-01-11 15:39:08 +11001836 kmem_zone_destroy(xfs_buf_zone);
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001837 out_free_trace_buf:
Nathan Scottce8e9222006-01-11 15:39:08 +11001838#ifdef XFS_BUF_TRACE
1839 ktrace_free(xfs_buf_trace_buf);
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001840#endif
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001841 return error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001842}
1843
Linus Torvalds1da177e2005-04-16 15:20:36 -07001844void
Nathan Scottce8e9222006-01-11 15:39:08 +11001845xfs_buf_terminate(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001846{
Nathan Scottce8e9222006-01-11 15:39:08 +11001847 kmem_shake_deregister(xfs_buf_shake);
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001848 destroy_workqueue(xfsdatad_workqueue);
1849 destroy_workqueue(xfslogd_workqueue);
Nathan Scottce8e9222006-01-11 15:39:08 +11001850 kmem_zone_destroy(xfs_buf_zone);
1851#ifdef XFS_BUF_TRACE
1852 ktrace_free(xfs_buf_trace_buf);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001853#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001854}