blob: 26fed0756f0100b5c2e2aec31eb5e1f5853c9ed4 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Nathan Scott7b718762005-11-02 14:58:39 +11002 * Copyright (c) 2000-2005 Silicon Graphics, Inc.
3 * All Rights Reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
Nathan Scott7b718762005-11-02 14:58:39 +11005 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License as
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 * published by the Free Software Foundation.
8 *
Nathan Scott7b718762005-11-02 14:58:39 +11009 * This program is distributed in the hope that it would be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 *
Nathan Scott7b718762005-11-02 14:58:39 +110014 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
Linus Torvalds1da177e2005-04-16 15:20:36 -070017 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/stddef.h>
19#include <linux/errno.h>
20#include <linux/slab.h>
21#include <linux/pagemap.h>
22#include <linux/init.h>
23#include <linux/vmalloc.h>
24#include <linux/bio.h>
25#include <linux/sysctl.h>
26#include <linux/proc_fs.h>
27#include <linux/workqueue.h>
28#include <linux/percpu.h>
29#include <linux/blkdev.h>
30#include <linux/hash.h>
Christoph Hellwig4df08c52005-09-05 08:34:18 +100031#include <linux/kthread.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080032#include <linux/migrate.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070033#include "xfs_linux.h"
34
Nathan Scottce8e9222006-01-11 15:39:08 +110035STATIC kmem_zone_t *xfs_buf_zone;
36STATIC kmem_shaker_t xfs_buf_shake;
David Chinnera6867a62006-01-11 15:37:58 +110037STATIC int xfsbufd(void *);
Al Viro27496a82005-10-21 03:20:48 -040038STATIC int xfsbufd_wakeup(int, gfp_t);
Nathan Scottce8e9222006-01-11 15:39:08 +110039STATIC void xfs_buf_delwri_queue(xfs_buf_t *, int);
Christoph Hellwig23ea4032005-06-21 15:14:01 +100040
41STATIC struct workqueue_struct *xfslogd_workqueue;
Christoph Hellwig0829c362005-09-02 16:58:49 +100042struct workqueue_struct *xfsdatad_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070043
Nathan Scottce8e9222006-01-11 15:39:08 +110044#ifdef XFS_BUF_TRACE
Linus Torvalds1da177e2005-04-16 15:20:36 -070045void
Nathan Scottce8e9222006-01-11 15:39:08 +110046xfs_buf_trace(
47 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -070048 char *id,
49 void *data,
50 void *ra)
51{
Nathan Scottce8e9222006-01-11 15:39:08 +110052 ktrace_enter(xfs_buf_trace_buf,
53 bp, id,
54 (void *)(unsigned long)bp->b_flags,
55 (void *)(unsigned long)bp->b_hold.counter,
56 (void *)(unsigned long)bp->b_sema.count.counter,
Linus Torvalds1da177e2005-04-16 15:20:36 -070057 (void *)current,
58 data, ra,
Nathan Scottce8e9222006-01-11 15:39:08 +110059 (void *)(unsigned long)((bp->b_file_offset>>32) & 0xffffffff),
60 (void *)(unsigned long)(bp->b_file_offset & 0xffffffff),
61 (void *)(unsigned long)bp->b_buffer_length,
Linus Torvalds1da177e2005-04-16 15:20:36 -070062 NULL, NULL, NULL, NULL, NULL);
63}
Nathan Scottce8e9222006-01-11 15:39:08 +110064ktrace_t *xfs_buf_trace_buf;
65#define XFS_BUF_TRACE_SIZE 4096
66#define XB_TRACE(bp, id, data) \
67 xfs_buf_trace(bp, id, (void *)data, (void *)__builtin_return_address(0))
Linus Torvalds1da177e2005-04-16 15:20:36 -070068#else
Nathan Scottce8e9222006-01-11 15:39:08 +110069#define XB_TRACE(bp, id, data) do { } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -070070#endif
71
Nathan Scottce8e9222006-01-11 15:39:08 +110072#ifdef XFS_BUF_LOCK_TRACKING
73# define XB_SET_OWNER(bp) ((bp)->b_last_holder = current->pid)
74# define XB_CLEAR_OWNER(bp) ((bp)->b_last_holder = -1)
75# define XB_GET_OWNER(bp) ((bp)->b_last_holder)
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#else
Nathan Scottce8e9222006-01-11 15:39:08 +110077# define XB_SET_OWNER(bp) do { } while (0)
78# define XB_CLEAR_OWNER(bp) do { } while (0)
79# define XB_GET_OWNER(bp) do { } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -070080#endif
81
Nathan Scottce8e9222006-01-11 15:39:08 +110082#define xb_to_gfp(flags) \
83 ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : \
84 ((flags) & XBF_DONT_BLOCK) ? GFP_NOFS : GFP_KERNEL) | __GFP_NOWARN)
Linus Torvalds1da177e2005-04-16 15:20:36 -070085
Nathan Scottce8e9222006-01-11 15:39:08 +110086#define xb_to_km(flags) \
87 (((flags) & XBF_DONT_BLOCK) ? KM_NOFS : KM_SLEEP)
Linus Torvalds1da177e2005-04-16 15:20:36 -070088
Nathan Scottce8e9222006-01-11 15:39:08 +110089#define xfs_buf_allocate(flags) \
90 kmem_zone_alloc(xfs_buf_zone, xb_to_km(flags))
91#define xfs_buf_deallocate(bp) \
92 kmem_zone_free(xfs_buf_zone, (bp));
Linus Torvalds1da177e2005-04-16 15:20:36 -070093
94/*
Nathan Scottce8e9222006-01-11 15:39:08 +110095 * Page Region interfaces.
Linus Torvalds1da177e2005-04-16 15:20:36 -070096 *
Nathan Scottce8e9222006-01-11 15:39:08 +110097 * For pages in filesystems where the blocksize is smaller than the
98 * pagesize, we use the page->private field (long) to hold a bitmap
99 * of uptodate regions within the page.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100 *
Nathan Scottce8e9222006-01-11 15:39:08 +1100101 * Each such region is "bytes per page / bits per long" bytes long.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102 *
Nathan Scottce8e9222006-01-11 15:39:08 +1100103 * NBPPR == number-of-bytes-per-page-region
104 * BTOPR == bytes-to-page-region (rounded up)
105 * BTOPRT == bytes-to-page-region-truncated (rounded down)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106 */
107#if (BITS_PER_LONG == 32)
108#define PRSHIFT (PAGE_CACHE_SHIFT - 5) /* (32 == 1<<5) */
109#elif (BITS_PER_LONG == 64)
110#define PRSHIFT (PAGE_CACHE_SHIFT - 6) /* (64 == 1<<6) */
111#else
112#error BITS_PER_LONG must be 32 or 64
113#endif
114#define NBPPR (PAGE_CACHE_SIZE/BITS_PER_LONG)
115#define BTOPR(b) (((unsigned int)(b) + (NBPPR - 1)) >> PRSHIFT)
116#define BTOPRT(b) (((unsigned int)(b) >> PRSHIFT))
117
118STATIC unsigned long
119page_region_mask(
120 size_t offset,
121 size_t length)
122{
123 unsigned long mask;
124 int first, final;
125
126 first = BTOPR(offset);
127 final = BTOPRT(offset + length - 1);
128 first = min(first, final);
129
130 mask = ~0UL;
131 mask <<= BITS_PER_LONG - (final - first);
132 mask >>= BITS_PER_LONG - (final);
133
134 ASSERT(offset + length <= PAGE_CACHE_SIZE);
135 ASSERT((final - first) < BITS_PER_LONG && (final - first) >= 0);
136
137 return mask;
138}
139
140STATIC inline void
141set_page_region(
142 struct page *page,
143 size_t offset,
144 size_t length)
145{
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700146 set_page_private(page,
147 page_private(page) | page_region_mask(offset, length));
148 if (page_private(page) == ~0UL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149 SetPageUptodate(page);
150}
151
152STATIC inline int
153test_page_region(
154 struct page *page,
155 size_t offset,
156 size_t length)
157{
158 unsigned long mask = page_region_mask(offset, length);
159
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700160 return (mask && (page_private(page) & mask) == mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161}
162
163/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100164 * Mapping of multi-page buffers into contiguous virtual space
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165 */
166
167typedef struct a_list {
168 void *vm_addr;
169 struct a_list *next;
170} a_list_t;
171
172STATIC a_list_t *as_free_head;
173STATIC int as_list_len;
174STATIC DEFINE_SPINLOCK(as_lock);
175
176/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100177 * Try to batch vunmaps because they are costly.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178 */
179STATIC void
180free_address(
181 void *addr)
182{
183 a_list_t *aentry;
184
Jeff Dike7b04d712006-04-10 22:53:27 -0700185 aentry = kmalloc(sizeof(a_list_t), GFP_NOWAIT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700186 if (likely(aentry)) {
187 spin_lock(&as_lock);
188 aentry->next = as_free_head;
189 aentry->vm_addr = addr;
190 as_free_head = aentry;
191 as_list_len++;
192 spin_unlock(&as_lock);
193 } else {
194 vunmap(addr);
195 }
196}
197
198STATIC void
199purge_addresses(void)
200{
201 a_list_t *aentry, *old;
202
203 if (as_free_head == NULL)
204 return;
205
206 spin_lock(&as_lock);
207 aentry = as_free_head;
208 as_free_head = NULL;
209 as_list_len = 0;
210 spin_unlock(&as_lock);
211
212 while ((old = aentry) != NULL) {
213 vunmap(aentry->vm_addr);
214 aentry = aentry->next;
215 kfree(old);
216 }
217}
218
219/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100220 * Internal xfs_buf_t object manipulation
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221 */
222
223STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +1100224_xfs_buf_initialize(
225 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700226 xfs_buftarg_t *target,
Nathan Scott204ab252006-01-11 20:50:22 +1100227 xfs_off_t range_base,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228 size_t range_length,
Nathan Scottce8e9222006-01-11 15:39:08 +1100229 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230{
231 /*
Nathan Scottce8e9222006-01-11 15:39:08 +1100232 * We don't want certain flags to appear in b_flags.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100234 flags &= ~(XBF_LOCK|XBF_MAPPED|XBF_DONT_BLOCK|XBF_READ_AHEAD);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700235
Nathan Scottce8e9222006-01-11 15:39:08 +1100236 memset(bp, 0, sizeof(xfs_buf_t));
237 atomic_set(&bp->b_hold, 1);
238 init_MUTEX_LOCKED(&bp->b_iodonesema);
239 INIT_LIST_HEAD(&bp->b_list);
240 INIT_LIST_HEAD(&bp->b_hash_list);
241 init_MUTEX_LOCKED(&bp->b_sema); /* held, no waiters */
242 XB_SET_OWNER(bp);
243 bp->b_target = target;
244 bp->b_file_offset = range_base;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245 /*
246 * Set buffer_length and count_desired to the same value initially.
247 * I/O routines should use count_desired, which will be the same in
248 * most cases but may be reset (e.g. XFS recovery).
249 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100250 bp->b_buffer_length = bp->b_count_desired = range_length;
251 bp->b_flags = flags;
252 bp->b_bn = XFS_BUF_DADDR_NULL;
253 atomic_set(&bp->b_pin_count, 0);
254 init_waitqueue_head(&bp->b_waiters);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255
Nathan Scottce8e9222006-01-11 15:39:08 +1100256 XFS_STATS_INC(xb_create);
257 XB_TRACE(bp, "initialize", target);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258}
259
260/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100261 * Allocate a page array capable of holding a specified number
262 * of pages, and point the page buf at it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263 */
264STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100265_xfs_buf_get_pages(
266 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700267 int page_count,
Nathan Scottce8e9222006-01-11 15:39:08 +1100268 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700269{
270 /* Make sure that we have a page list */
Nathan Scottce8e9222006-01-11 15:39:08 +1100271 if (bp->b_pages == NULL) {
272 bp->b_offset = xfs_buf_poff(bp->b_file_offset);
273 bp->b_page_count = page_count;
274 if (page_count <= XB_PAGES) {
275 bp->b_pages = bp->b_page_array;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100277 bp->b_pages = kmem_alloc(sizeof(struct page *) *
278 page_count, xb_to_km(flags));
279 if (bp->b_pages == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700280 return -ENOMEM;
281 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100282 memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700283 }
284 return 0;
285}
286
287/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100288 * Frees b_pages if it was allocated.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700289 */
290STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +1100291_xfs_buf_free_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292 xfs_buf_t *bp)
293{
Nathan Scottce8e9222006-01-11 15:39:08 +1100294 if (bp->b_pages != bp->b_page_array) {
295 kmem_free(bp->b_pages,
296 bp->b_page_count * sizeof(struct page *));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297 }
298}
299
300/*
301 * Releases the specified buffer.
302 *
303 * The modification state of any associated pages is left unchanged.
Nathan Scottce8e9222006-01-11 15:39:08 +1100304 * The buffer most not be on any hash - use xfs_buf_rele instead for
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305 * hashed and refcounted buffers
306 */
307void
Nathan Scottce8e9222006-01-11 15:39:08 +1100308xfs_buf_free(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309 xfs_buf_t *bp)
310{
Nathan Scottce8e9222006-01-11 15:39:08 +1100311 XB_TRACE(bp, "free", 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312
Nathan Scottce8e9222006-01-11 15:39:08 +1100313 ASSERT(list_empty(&bp->b_hash_list));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314
Nathan Scottce8e9222006-01-11 15:39:08 +1100315 if (bp->b_flags & _XBF_PAGE_CACHE) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700316 uint i;
317
Nathan Scottce8e9222006-01-11 15:39:08 +1100318 if ((bp->b_flags & XBF_MAPPED) && (bp->b_page_count > 1))
319 free_address(bp->b_addr - bp->b_offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700320
Nathan Scottce8e9222006-01-11 15:39:08 +1100321 for (i = 0; i < bp->b_page_count; i++)
322 page_cache_release(bp->b_pages[i]);
323 _xfs_buf_free_pages(bp);
324 } else if (bp->b_flags & _XBF_KMEM_ALLOC) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325 /*
Nathan Scottce8e9222006-01-11 15:39:08 +1100326 * XXX(hch): bp->b_count_desired might be incorrect (see
327 * xfs_buf_associate_memory for details), but fortunately
Linus Torvalds1da177e2005-04-16 15:20:36 -0700328 * the Linux version of kmem_free ignores the len argument..
329 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100330 kmem_free(bp->b_addr, bp->b_count_desired);
331 _xfs_buf_free_pages(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332 }
333
Nathan Scottce8e9222006-01-11 15:39:08 +1100334 xfs_buf_deallocate(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700335}
336
337/*
338 * Finds all pages for buffer in question and builds it's page list.
339 */
340STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100341_xfs_buf_lookup_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700342 xfs_buf_t *bp,
343 uint flags)
344{
Nathan Scottce8e9222006-01-11 15:39:08 +1100345 struct address_space *mapping = bp->b_target->bt_mapping;
346 size_t blocksize = bp->b_target->bt_bsize;
347 size_t size = bp->b_count_desired;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348 size_t nbytes, offset;
Nathan Scottce8e9222006-01-11 15:39:08 +1100349 gfp_t gfp_mask = xb_to_gfp(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350 unsigned short page_count, i;
351 pgoff_t first;
Nathan Scott204ab252006-01-11 20:50:22 +1100352 xfs_off_t end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353 int error;
354
Nathan Scottce8e9222006-01-11 15:39:08 +1100355 end = bp->b_file_offset + bp->b_buffer_length;
356 page_count = xfs_buf_btoc(end) - xfs_buf_btoct(bp->b_file_offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357
Nathan Scottce8e9222006-01-11 15:39:08 +1100358 error = _xfs_buf_get_pages(bp, page_count, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700359 if (unlikely(error))
360 return error;
Nathan Scottce8e9222006-01-11 15:39:08 +1100361 bp->b_flags |= _XBF_PAGE_CACHE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362
Nathan Scottce8e9222006-01-11 15:39:08 +1100363 offset = bp->b_offset;
364 first = bp->b_file_offset >> PAGE_CACHE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700365
Nathan Scottce8e9222006-01-11 15:39:08 +1100366 for (i = 0; i < bp->b_page_count; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700367 struct page *page;
368 uint retries = 0;
369
370 retry:
371 page = find_or_create_page(mapping, first + i, gfp_mask);
372 if (unlikely(page == NULL)) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100373 if (flags & XBF_READ_AHEAD) {
374 bp->b_page_count = i;
375 for (i = 0; i < bp->b_page_count; i++)
376 unlock_page(bp->b_pages[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377 return -ENOMEM;
378 }
379
380 /*
381 * This could deadlock.
382 *
383 * But until all the XFS lowlevel code is revamped to
384 * handle buffer allocation failures we can't do much.
385 */
386 if (!(++retries % 100))
387 printk(KERN_ERR
388 "XFS: possible memory allocation "
389 "deadlock in %s (mode:0x%x)\n",
390 __FUNCTION__, gfp_mask);
391
Nathan Scottce8e9222006-01-11 15:39:08 +1100392 XFS_STATS_INC(xb_page_retries);
Christoph Hellwig23ea4032005-06-21 15:14:01 +1000393 xfsbufd_wakeup(0, gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394 blk_congestion_wait(WRITE, HZ/50);
395 goto retry;
396 }
397
Nathan Scottce8e9222006-01-11 15:39:08 +1100398 XFS_STATS_INC(xb_page_found);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700399
400 nbytes = min_t(size_t, size, PAGE_CACHE_SIZE - offset);
401 size -= nbytes;
402
403 if (!PageUptodate(page)) {
404 page_count--;
405 if (blocksize >= PAGE_CACHE_SIZE) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100406 if (flags & XBF_READ)
407 bp->b_locked = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408 } else if (!PagePrivate(page)) {
409 if (test_page_region(page, offset, nbytes))
410 page_count++;
411 }
412 }
413
Nathan Scottce8e9222006-01-11 15:39:08 +1100414 bp->b_pages[i] = page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415 offset = 0;
416 }
417
Nathan Scottce8e9222006-01-11 15:39:08 +1100418 if (!bp->b_locked) {
419 for (i = 0; i < bp->b_page_count; i++)
420 unlock_page(bp->b_pages[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700421 }
422
Nathan Scottce8e9222006-01-11 15:39:08 +1100423 if (page_count == bp->b_page_count)
424 bp->b_flags |= XBF_DONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700425
Nathan Scottce8e9222006-01-11 15:39:08 +1100426 XB_TRACE(bp, "lookup_pages", (long)page_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427 return error;
428}
429
430/*
431 * Map buffer into kernel address-space if nessecary.
432 */
433STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100434_xfs_buf_map_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435 xfs_buf_t *bp,
436 uint flags)
437{
438 /* A single page buffer is always mappable */
Nathan Scottce8e9222006-01-11 15:39:08 +1100439 if (bp->b_page_count == 1) {
440 bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
441 bp->b_flags |= XBF_MAPPED;
442 } else if (flags & XBF_MAPPED) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443 if (as_list_len > 64)
444 purge_addresses();
Nathan Scottce8e9222006-01-11 15:39:08 +1100445 bp->b_addr = vmap(bp->b_pages, bp->b_page_count,
446 VM_MAP, PAGE_KERNEL);
447 if (unlikely(bp->b_addr == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700448 return -ENOMEM;
Nathan Scottce8e9222006-01-11 15:39:08 +1100449 bp->b_addr += bp->b_offset;
450 bp->b_flags |= XBF_MAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700451 }
452
453 return 0;
454}
455
456/*
457 * Finding and Reading Buffers
458 */
459
460/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100461 * Look up, and creates if absent, a lockable buffer for
Linus Torvalds1da177e2005-04-16 15:20:36 -0700462 * a given range of an inode. The buffer is returned
463 * locked. If other overlapping buffers exist, they are
464 * released before the new buffer is created and locked,
465 * which may imply that this call will block until those buffers
466 * are unlocked. No I/O is implied by this call.
467 */
468xfs_buf_t *
Nathan Scottce8e9222006-01-11 15:39:08 +1100469_xfs_buf_find(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700470 xfs_buftarg_t *btp, /* block device target */
Nathan Scott204ab252006-01-11 20:50:22 +1100471 xfs_off_t ioff, /* starting offset of range */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472 size_t isize, /* length of range */
Nathan Scottce8e9222006-01-11 15:39:08 +1100473 xfs_buf_flags_t flags,
474 xfs_buf_t *new_bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700475{
Nathan Scott204ab252006-01-11 20:50:22 +1100476 xfs_off_t range_base;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700477 size_t range_length;
478 xfs_bufhash_t *hash;
Nathan Scottce8e9222006-01-11 15:39:08 +1100479 xfs_buf_t *bp, *n;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700480
481 range_base = (ioff << BBSHIFT);
482 range_length = (isize << BBSHIFT);
483
484 /* Check for IOs smaller than the sector size / not sector aligned */
Nathan Scottce8e9222006-01-11 15:39:08 +1100485 ASSERT(!(range_length < (1 << btp->bt_sshift)));
Nathan Scott204ab252006-01-11 20:50:22 +1100486 ASSERT(!(range_base & (xfs_off_t)btp->bt_smask));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700487
488 hash = &btp->bt_hash[hash_long((unsigned long)ioff, btp->bt_hashshift)];
489
490 spin_lock(&hash->bh_lock);
491
Nathan Scottce8e9222006-01-11 15:39:08 +1100492 list_for_each_entry_safe(bp, n, &hash->bh_list, b_hash_list) {
493 ASSERT(btp == bp->b_target);
494 if (bp->b_file_offset == range_base &&
495 bp->b_buffer_length == range_length) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700496 /*
Nathan Scottce8e9222006-01-11 15:39:08 +1100497 * If we look at something, bring it to the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498 * front of the list for next time.
499 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100500 atomic_inc(&bp->b_hold);
501 list_move(&bp->b_hash_list, &hash->bh_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502 goto found;
503 }
504 }
505
506 /* No match found */
Nathan Scottce8e9222006-01-11 15:39:08 +1100507 if (new_bp) {
508 _xfs_buf_initialize(new_bp, btp, range_base,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700509 range_length, flags);
Nathan Scottce8e9222006-01-11 15:39:08 +1100510 new_bp->b_hash = hash;
511 list_add(&new_bp->b_hash_list, &hash->bh_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700512 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100513 XFS_STATS_INC(xb_miss_locked);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700514 }
515
516 spin_unlock(&hash->bh_lock);
Nathan Scottce8e9222006-01-11 15:39:08 +1100517 return new_bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518
519found:
520 spin_unlock(&hash->bh_lock);
521
522 /* Attempt to get the semaphore without sleeping,
523 * if this does not work then we need to drop the
524 * spinlock and do a hard attempt on the semaphore.
525 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100526 if (down_trylock(&bp->b_sema)) {
527 if (!(flags & XBF_TRYLOCK)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528 /* wait for buffer ownership */
Nathan Scottce8e9222006-01-11 15:39:08 +1100529 XB_TRACE(bp, "get_lock", 0);
530 xfs_buf_lock(bp);
531 XFS_STATS_INC(xb_get_locked_waited);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700532 } else {
533 /* We asked for a trylock and failed, no need
534 * to look at file offset and length here, we
Nathan Scottce8e9222006-01-11 15:39:08 +1100535 * know that this buffer at least overlaps our
536 * buffer and is locked, therefore our buffer
537 * either does not exist, or is this buffer.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700538 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100539 xfs_buf_rele(bp);
540 XFS_STATS_INC(xb_busy_locked);
541 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542 }
543 } else {
544 /* trylock worked */
Nathan Scottce8e9222006-01-11 15:39:08 +1100545 XB_SET_OWNER(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700546 }
547
Nathan Scottce8e9222006-01-11 15:39:08 +1100548 if (bp->b_flags & XBF_STALE) {
549 ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
550 bp->b_flags &= XBF_MAPPED;
David Chinner2f926582005-09-05 08:33:35 +1000551 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100552 XB_TRACE(bp, "got_lock", 0);
553 XFS_STATS_INC(xb_get_locked);
554 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700555}
556
557/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100558 * Assembles a buffer covering the specified range.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700559 * Storage in memory for all portions of the buffer will be allocated,
560 * although backing storage may not be.
561 */
562xfs_buf_t *
Nathan Scottce8e9222006-01-11 15:39:08 +1100563xfs_buf_get_flags(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700564 xfs_buftarg_t *target,/* target for buffer */
Nathan Scott204ab252006-01-11 20:50:22 +1100565 xfs_off_t ioff, /* starting offset of range */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700566 size_t isize, /* length of range */
Nathan Scottce8e9222006-01-11 15:39:08 +1100567 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700568{
Nathan Scottce8e9222006-01-11 15:39:08 +1100569 xfs_buf_t *bp, *new_bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570 int error = 0, i;
571
Nathan Scottce8e9222006-01-11 15:39:08 +1100572 new_bp = xfs_buf_allocate(flags);
573 if (unlikely(!new_bp))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574 return NULL;
575
Nathan Scottce8e9222006-01-11 15:39:08 +1100576 bp = _xfs_buf_find(target, ioff, isize, flags, new_bp);
577 if (bp == new_bp) {
578 error = _xfs_buf_lookup_pages(bp, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700579 if (error)
580 goto no_buffer;
581 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100582 xfs_buf_deallocate(new_bp);
583 if (unlikely(bp == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584 return NULL;
585 }
586
Nathan Scottce8e9222006-01-11 15:39:08 +1100587 for (i = 0; i < bp->b_page_count; i++)
588 mark_page_accessed(bp->b_pages[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589
Nathan Scottce8e9222006-01-11 15:39:08 +1100590 if (!(bp->b_flags & XBF_MAPPED)) {
591 error = _xfs_buf_map_pages(bp, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592 if (unlikely(error)) {
593 printk(KERN_WARNING "%s: failed to map pages\n",
594 __FUNCTION__);
595 goto no_buffer;
596 }
597 }
598
Nathan Scottce8e9222006-01-11 15:39:08 +1100599 XFS_STATS_INC(xb_get);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600
601 /*
602 * Always fill in the block number now, the mapped cases can do
603 * their own overlay of this later.
604 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100605 bp->b_bn = ioff;
606 bp->b_count_desired = bp->b_buffer_length;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607
Nathan Scottce8e9222006-01-11 15:39:08 +1100608 XB_TRACE(bp, "get", (unsigned long)flags);
609 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700610
611 no_buffer:
Nathan Scottce8e9222006-01-11 15:39:08 +1100612 if (flags & (XBF_LOCK | XBF_TRYLOCK))
613 xfs_buf_unlock(bp);
614 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615 return NULL;
616}
617
618xfs_buf_t *
619xfs_buf_read_flags(
620 xfs_buftarg_t *target,
Nathan Scott204ab252006-01-11 20:50:22 +1100621 xfs_off_t ioff,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622 size_t isize,
Nathan Scottce8e9222006-01-11 15:39:08 +1100623 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624{
Nathan Scottce8e9222006-01-11 15:39:08 +1100625 xfs_buf_t *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700626
Nathan Scottce8e9222006-01-11 15:39:08 +1100627 flags |= XBF_READ;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700628
Nathan Scottce8e9222006-01-11 15:39:08 +1100629 bp = xfs_buf_get_flags(target, ioff, isize, flags);
630 if (bp) {
631 if (!XFS_BUF_ISDONE(bp)) {
632 XB_TRACE(bp, "read", (unsigned long)flags);
633 XFS_STATS_INC(xb_get_read);
634 xfs_buf_iostart(bp, flags);
635 } else if (flags & XBF_ASYNC) {
636 XB_TRACE(bp, "read_async", (unsigned long)flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700637 /*
638 * Read ahead call which is already satisfied,
639 * drop the buffer
640 */
641 goto no_buffer;
642 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100643 XB_TRACE(bp, "read_done", (unsigned long)flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644 /* We do not want read in the flags */
Nathan Scottce8e9222006-01-11 15:39:08 +1100645 bp->b_flags &= ~XBF_READ;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700646 }
647 }
648
Nathan Scottce8e9222006-01-11 15:39:08 +1100649 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700650
651 no_buffer:
Nathan Scottce8e9222006-01-11 15:39:08 +1100652 if (flags & (XBF_LOCK | XBF_TRYLOCK))
653 xfs_buf_unlock(bp);
654 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700655 return NULL;
656}
657
658/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100659 * If we are not low on memory then do the readahead in a deadlock
660 * safe manner.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700661 */
662void
Nathan Scottce8e9222006-01-11 15:39:08 +1100663xfs_buf_readahead(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700664 xfs_buftarg_t *target,
Nathan Scott204ab252006-01-11 20:50:22 +1100665 xfs_off_t ioff,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666 size_t isize,
Nathan Scottce8e9222006-01-11 15:39:08 +1100667 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668{
669 struct backing_dev_info *bdi;
670
Nathan Scottce8e9222006-01-11 15:39:08 +1100671 bdi = target->bt_mapping->backing_dev_info;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700672 if (bdi_read_congested(bdi))
673 return;
674
Nathan Scottce8e9222006-01-11 15:39:08 +1100675 flags |= (XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700676 xfs_buf_read_flags(target, ioff, isize, flags);
677}
678
679xfs_buf_t *
Nathan Scottce8e9222006-01-11 15:39:08 +1100680xfs_buf_get_empty(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700681 size_t len,
682 xfs_buftarg_t *target)
683{
Nathan Scottce8e9222006-01-11 15:39:08 +1100684 xfs_buf_t *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700685
Nathan Scottce8e9222006-01-11 15:39:08 +1100686 bp = xfs_buf_allocate(0);
687 if (bp)
688 _xfs_buf_initialize(bp, target, 0, len, 0);
689 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700690}
691
692static inline struct page *
693mem_to_page(
694 void *addr)
695{
696 if (((unsigned long)addr < VMALLOC_START) ||
697 ((unsigned long)addr >= VMALLOC_END)) {
698 return virt_to_page(addr);
699 } else {
700 return vmalloc_to_page(addr);
701 }
702}
703
704int
Nathan Scottce8e9222006-01-11 15:39:08 +1100705xfs_buf_associate_memory(
706 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700707 void *mem,
708 size_t len)
709{
710 int rval;
711 int i = 0;
712 size_t ptr;
713 size_t end, end_cur;
714 off_t offset;
715 int page_count;
716
717 page_count = PAGE_CACHE_ALIGN(len) >> PAGE_CACHE_SHIFT;
718 offset = (off_t) mem - ((off_t)mem & PAGE_CACHE_MASK);
719 if (offset && (len > PAGE_CACHE_SIZE))
720 page_count++;
721
722 /* Free any previous set of page pointers */
Nathan Scottce8e9222006-01-11 15:39:08 +1100723 if (bp->b_pages)
724 _xfs_buf_free_pages(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700725
Nathan Scottce8e9222006-01-11 15:39:08 +1100726 bp->b_pages = NULL;
727 bp->b_addr = mem;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700728
Nathan Scottce8e9222006-01-11 15:39:08 +1100729 rval = _xfs_buf_get_pages(bp, page_count, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700730 if (rval)
731 return rval;
732
Nathan Scottce8e9222006-01-11 15:39:08 +1100733 bp->b_offset = offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734 ptr = (size_t) mem & PAGE_CACHE_MASK;
735 end = PAGE_CACHE_ALIGN((size_t) mem + len);
736 end_cur = end;
737 /* set up first page */
Nathan Scottce8e9222006-01-11 15:39:08 +1100738 bp->b_pages[0] = mem_to_page(mem);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739
740 ptr += PAGE_CACHE_SIZE;
Nathan Scottce8e9222006-01-11 15:39:08 +1100741 bp->b_page_count = ++i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700742 while (ptr < end) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100743 bp->b_pages[i] = mem_to_page((void *)ptr);
744 bp->b_page_count = ++i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700745 ptr += PAGE_CACHE_SIZE;
746 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100747 bp->b_locked = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700748
Nathan Scottce8e9222006-01-11 15:39:08 +1100749 bp->b_count_desired = bp->b_buffer_length = len;
750 bp->b_flags |= XBF_MAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700751
752 return 0;
753}
754
755xfs_buf_t *
Nathan Scottce8e9222006-01-11 15:39:08 +1100756xfs_buf_get_noaddr(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700757 size_t len,
758 xfs_buftarg_t *target)
759{
760 size_t malloc_len = len;
761 xfs_buf_t *bp;
762 void *data;
763 int error;
764
Nathan Scottce8e9222006-01-11 15:39:08 +1100765 bp = xfs_buf_allocate(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700766 if (unlikely(bp == NULL))
767 goto fail;
Nathan Scottce8e9222006-01-11 15:39:08 +1100768 _xfs_buf_initialize(bp, target, 0, len, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700769
770 try_again:
771 data = kmem_alloc(malloc_len, KM_SLEEP | KM_MAYFAIL);
772 if (unlikely(data == NULL))
773 goto fail_free_buf;
774
775 /* check whether alignment matches.. */
776 if ((__psunsigned_t)data !=
Nathan Scottce8e9222006-01-11 15:39:08 +1100777 ((__psunsigned_t)data & ~target->bt_smask)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700778 /* .. else double the size and try again */
779 kmem_free(data, malloc_len);
780 malloc_len <<= 1;
781 goto try_again;
782 }
783
Nathan Scottce8e9222006-01-11 15:39:08 +1100784 error = xfs_buf_associate_memory(bp, data, len);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700785 if (error)
786 goto fail_free_mem;
Nathan Scottce8e9222006-01-11 15:39:08 +1100787 bp->b_flags |= _XBF_KMEM_ALLOC;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700788
Nathan Scottce8e9222006-01-11 15:39:08 +1100789 xfs_buf_unlock(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700790
Nathan Scottce8e9222006-01-11 15:39:08 +1100791 XB_TRACE(bp, "no_daddr", data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700792 return bp;
793 fail_free_mem:
794 kmem_free(data, malloc_len);
795 fail_free_buf:
Nathan Scottce8e9222006-01-11 15:39:08 +1100796 xfs_buf_free(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797 fail:
798 return NULL;
799}
800
801/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802 * Increment reference count on buffer, to hold the buffer concurrently
803 * with another thread which may release (free) the buffer asynchronously.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700804 * Must hold the buffer already to call this function.
805 */
806void
Nathan Scottce8e9222006-01-11 15:39:08 +1100807xfs_buf_hold(
808 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809{
Nathan Scottce8e9222006-01-11 15:39:08 +1100810 atomic_inc(&bp->b_hold);
811 XB_TRACE(bp, "hold", 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812}
813
814/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100815 * Releases a hold on the specified buffer. If the
816 * the hold count is 1, calls xfs_buf_free.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700817 */
818void
Nathan Scottce8e9222006-01-11 15:39:08 +1100819xfs_buf_rele(
820 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821{
Nathan Scottce8e9222006-01-11 15:39:08 +1100822 xfs_bufhash_t *hash = bp->b_hash;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700823
Nathan Scottce8e9222006-01-11 15:39:08 +1100824 XB_TRACE(bp, "rele", bp->b_relse);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700825
Nathan Scottfad3aa12006-02-01 12:14:52 +1100826 if (unlikely(!hash)) {
827 ASSERT(!bp->b_relse);
828 if (atomic_dec_and_test(&bp->b_hold))
829 xfs_buf_free(bp);
830 return;
831 }
832
Nathan Scottce8e9222006-01-11 15:39:08 +1100833 if (atomic_dec_and_lock(&bp->b_hold, &hash->bh_lock)) {
834 if (bp->b_relse) {
835 atomic_inc(&bp->b_hold);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700836 spin_unlock(&hash->bh_lock);
Nathan Scottce8e9222006-01-11 15:39:08 +1100837 (*(bp->b_relse)) (bp);
838 } else if (bp->b_flags & XBF_FS_MANAGED) {
Christoph Hellwig7f14d0a2005-11-02 15:09:35 +1100839 spin_unlock(&hash->bh_lock);
840 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100841 ASSERT(!(bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)));
842 list_del_init(&bp->b_hash_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700843 spin_unlock(&hash->bh_lock);
Nathan Scottce8e9222006-01-11 15:39:08 +1100844 xfs_buf_free(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700845 }
David Chinner2f926582005-09-05 08:33:35 +1000846 } else {
847 /*
848 * Catch reference count leaks
849 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100850 ASSERT(atomic_read(&bp->b_hold) >= 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700851 }
852}
853
854
855/*
856 * Mutual exclusion on buffers. Locking model:
857 *
858 * Buffers associated with inodes for which buffer locking
859 * is not enabled are not protected by semaphores, and are
860 * assumed to be exclusively owned by the caller. There is a
861 * spinlock in the buffer, used by the caller when concurrent
862 * access is possible.
863 */
864
865/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100866 * Locks a buffer object, if it is not already locked.
867 * Note that this in no way locks the underlying pages, so it is only
868 * useful for synchronizing concurrent use of buffer objects, not for
869 * synchronizing independent access to the underlying pages.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870 */
871int
Nathan Scottce8e9222006-01-11 15:39:08 +1100872xfs_buf_cond_lock(
873 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700874{
875 int locked;
876
Nathan Scottce8e9222006-01-11 15:39:08 +1100877 locked = down_trylock(&bp->b_sema) == 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700878 if (locked) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100879 XB_SET_OWNER(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700880 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100881 XB_TRACE(bp, "cond_lock", (long)locked);
882 return locked ? 0 : -EBUSY;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700883}
884
885#if defined(DEBUG) || defined(XFS_BLI_TRACE)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700886int
Nathan Scottce8e9222006-01-11 15:39:08 +1100887xfs_buf_lock_value(
888 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889{
Nathan Scottce8e9222006-01-11 15:39:08 +1100890 return atomic_read(&bp->b_sema.count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700891}
892#endif
893
894/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100895 * Locks a buffer object.
896 * Note that this in no way locks the underlying pages, so it is only
897 * useful for synchronizing concurrent use of buffer objects, not for
898 * synchronizing independent access to the underlying pages.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700899 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100900void
901xfs_buf_lock(
902 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903{
Nathan Scottce8e9222006-01-11 15:39:08 +1100904 XB_TRACE(bp, "lock", 0);
905 if (atomic_read(&bp->b_io_remaining))
906 blk_run_address_space(bp->b_target->bt_mapping);
907 down(&bp->b_sema);
908 XB_SET_OWNER(bp);
909 XB_TRACE(bp, "locked", 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700910}
911
912/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100913 * Releases the lock on the buffer object.
David Chinner2f926582005-09-05 08:33:35 +1000914 * If the buffer is marked delwri but is not queued, do so before we
Nathan Scottce8e9222006-01-11 15:39:08 +1100915 * unlock the buffer as we need to set flags correctly. We also need to
David Chinner2f926582005-09-05 08:33:35 +1000916 * take a reference for the delwri queue because the unlocker is going to
917 * drop their's and they don't know we just queued it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700918 */
919void
Nathan Scottce8e9222006-01-11 15:39:08 +1100920xfs_buf_unlock(
921 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700922{
Nathan Scottce8e9222006-01-11 15:39:08 +1100923 if ((bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)) == XBF_DELWRI) {
924 atomic_inc(&bp->b_hold);
925 bp->b_flags |= XBF_ASYNC;
926 xfs_buf_delwri_queue(bp, 0);
David Chinner2f926582005-09-05 08:33:35 +1000927 }
928
Nathan Scottce8e9222006-01-11 15:39:08 +1100929 XB_CLEAR_OWNER(bp);
930 up(&bp->b_sema);
931 XB_TRACE(bp, "unlock", 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700932}
933
934
935/*
936 * Pinning Buffer Storage in Memory
Nathan Scottce8e9222006-01-11 15:39:08 +1100937 * Ensure that no attempt to force a buffer to disk will succeed.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700938 */
939void
Nathan Scottce8e9222006-01-11 15:39:08 +1100940xfs_buf_pin(
941 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700942{
Nathan Scottce8e9222006-01-11 15:39:08 +1100943 atomic_inc(&bp->b_pin_count);
944 XB_TRACE(bp, "pin", (long)bp->b_pin_count.counter);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700945}
946
Linus Torvalds1da177e2005-04-16 15:20:36 -0700947void
Nathan Scottce8e9222006-01-11 15:39:08 +1100948xfs_buf_unpin(
949 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950{
Nathan Scottce8e9222006-01-11 15:39:08 +1100951 if (atomic_dec_and_test(&bp->b_pin_count))
952 wake_up_all(&bp->b_waiters);
953 XB_TRACE(bp, "unpin", (long)bp->b_pin_count.counter);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700954}
955
956int
Nathan Scottce8e9222006-01-11 15:39:08 +1100957xfs_buf_ispin(
958 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959{
Nathan Scottce8e9222006-01-11 15:39:08 +1100960 return atomic_read(&bp->b_pin_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700961}
962
Nathan Scottce8e9222006-01-11 15:39:08 +1100963STATIC void
964xfs_buf_wait_unpin(
965 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700966{
967 DECLARE_WAITQUEUE (wait, current);
968
Nathan Scottce8e9222006-01-11 15:39:08 +1100969 if (atomic_read(&bp->b_pin_count) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700970 return;
971
Nathan Scottce8e9222006-01-11 15:39:08 +1100972 add_wait_queue(&bp->b_waiters, &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700973 for (;;) {
974 set_current_state(TASK_UNINTERRUPTIBLE);
Nathan Scottce8e9222006-01-11 15:39:08 +1100975 if (atomic_read(&bp->b_pin_count) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700976 break;
Nathan Scottce8e9222006-01-11 15:39:08 +1100977 if (atomic_read(&bp->b_io_remaining))
978 blk_run_address_space(bp->b_target->bt_mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700979 schedule();
980 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100981 remove_wait_queue(&bp->b_waiters, &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982 set_current_state(TASK_RUNNING);
983}
984
985/*
986 * Buffer Utility Routines
987 */
988
Linus Torvalds1da177e2005-04-16 15:20:36 -0700989STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +1100990xfs_buf_iodone_work(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700991 void *v)
992{
993 xfs_buf_t *bp = (xfs_buf_t *)v;
994
Nathan Scottce8e9222006-01-11 15:39:08 +1100995 if (bp->b_iodone)
996 (*(bp->b_iodone))(bp);
997 else if (bp->b_flags & XBF_ASYNC)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700998 xfs_buf_relse(bp);
999}
1000
1001void
Nathan Scottce8e9222006-01-11 15:39:08 +11001002xfs_buf_ioend(
1003 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001004 int schedule)
1005{
Nathan Scottce8e9222006-01-11 15:39:08 +11001006 bp->b_flags &= ~(XBF_READ | XBF_WRITE);
1007 if (bp->b_error == 0)
1008 bp->b_flags |= XBF_DONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001009
Nathan Scottce8e9222006-01-11 15:39:08 +11001010 XB_TRACE(bp, "iodone", bp->b_iodone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001011
Nathan Scottce8e9222006-01-11 15:39:08 +11001012 if ((bp->b_iodone) || (bp->b_flags & XBF_ASYNC)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001013 if (schedule) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001014 INIT_WORK(&bp->b_iodone_work, xfs_buf_iodone_work, bp);
1015 queue_work(xfslogd_workqueue, &bp->b_iodone_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001016 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +11001017 xfs_buf_iodone_work(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001018 }
1019 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +11001020 up(&bp->b_iodonesema);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001021 }
1022}
1023
Linus Torvalds1da177e2005-04-16 15:20:36 -07001024void
Nathan Scottce8e9222006-01-11 15:39:08 +11001025xfs_buf_ioerror(
1026 xfs_buf_t *bp,
1027 int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001028{
1029 ASSERT(error >= 0 && error <= 0xffff);
Nathan Scottce8e9222006-01-11 15:39:08 +11001030 bp->b_error = (unsigned short)error;
1031 XB_TRACE(bp, "ioerror", (unsigned long)error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001032}
1033
1034/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001035 * Initiate I/O on a buffer, based on the flags supplied.
1036 * The b_iodone routine in the buffer supplied will only be called
Linus Torvalds1da177e2005-04-16 15:20:36 -07001037 * when all of the subsidiary I/O requests, if any, have been completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001038 */
1039int
Nathan Scottce8e9222006-01-11 15:39:08 +11001040xfs_buf_iostart(
1041 xfs_buf_t *bp,
1042 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001043{
1044 int status = 0;
1045
Nathan Scottce8e9222006-01-11 15:39:08 +11001046 XB_TRACE(bp, "iostart", (unsigned long)flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001047
Nathan Scottce8e9222006-01-11 15:39:08 +11001048 if (flags & XBF_DELWRI) {
1049 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_ASYNC);
1050 bp->b_flags |= flags & (XBF_DELWRI | XBF_ASYNC);
1051 xfs_buf_delwri_queue(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001052 return status;
1053 }
1054
Nathan Scottce8e9222006-01-11 15:39:08 +11001055 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_ASYNC | XBF_DELWRI | \
1056 XBF_READ_AHEAD | _XBF_RUN_QUEUES);
1057 bp->b_flags |= flags & (XBF_READ | XBF_WRITE | XBF_ASYNC | \
1058 XBF_READ_AHEAD | _XBF_RUN_QUEUES);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001059
Nathan Scottce8e9222006-01-11 15:39:08 +11001060 BUG_ON(bp->b_bn == XFS_BUF_DADDR_NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001061
1062 /* For writes allow an alternate strategy routine to precede
1063 * the actual I/O request (which may not be issued at all in
1064 * a shutdown situation, for example).
1065 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001066 status = (flags & XBF_WRITE) ?
1067 xfs_buf_iostrategy(bp) : xfs_buf_iorequest(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001068
1069 /* Wait for I/O if we are not an async request.
1070 * Note: async I/O request completion will release the buffer,
1071 * and that can already be done by this point. So using the
1072 * buffer pointer from here on, after async I/O, is invalid.
1073 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001074 if (!status && !(flags & XBF_ASYNC))
1075 status = xfs_buf_iowait(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001076
1077 return status;
1078}
1079
Linus Torvalds1da177e2005-04-16 15:20:36 -07001080STATIC __inline__ int
Nathan Scottce8e9222006-01-11 15:39:08 +11001081_xfs_buf_iolocked(
1082 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001083{
Nathan Scottce8e9222006-01-11 15:39:08 +11001084 ASSERT(bp->b_flags & (XBF_READ | XBF_WRITE));
1085 if (bp->b_flags & XBF_READ)
1086 return bp->b_locked;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001087 return 0;
1088}
1089
1090STATIC __inline__ void
Nathan Scottce8e9222006-01-11 15:39:08 +11001091_xfs_buf_ioend(
1092 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001093 int schedule)
1094{
Nathan Scottce8e9222006-01-11 15:39:08 +11001095 if (atomic_dec_and_test(&bp->b_io_remaining) == 1) {
1096 bp->b_locked = 0;
1097 xfs_buf_ioend(bp, schedule);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001098 }
1099}
1100
1101STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +11001102xfs_buf_bio_end_io(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001103 struct bio *bio,
1104 unsigned int bytes_done,
1105 int error)
1106{
Nathan Scottce8e9222006-01-11 15:39:08 +11001107 xfs_buf_t *bp = (xfs_buf_t *)bio->bi_private;
1108 unsigned int blocksize = bp->b_target->bt_bsize;
Nathan Scotteedb5532005-09-02 16:39:56 +10001109 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001110
1111 if (bio->bi_size)
1112 return 1;
1113
1114 if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
Nathan Scottce8e9222006-01-11 15:39:08 +11001115 bp->b_error = EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001116
Nathan Scotteedb5532005-09-02 16:39:56 +10001117 do {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001118 struct page *page = bvec->bv_page;
1119
Nathan Scottce8e9222006-01-11 15:39:08 +11001120 if (unlikely(bp->b_error)) {
1121 if (bp->b_flags & XBF_READ)
Nathan Scotteedb5532005-09-02 16:39:56 +10001122 ClearPageUptodate(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001123 SetPageError(page);
Nathan Scottce8e9222006-01-11 15:39:08 +11001124 } else if (blocksize >= PAGE_CACHE_SIZE) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001125 SetPageUptodate(page);
1126 } else if (!PagePrivate(page) &&
Nathan Scottce8e9222006-01-11 15:39:08 +11001127 (bp->b_flags & _XBF_PAGE_CACHE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001128 set_page_region(page, bvec->bv_offset, bvec->bv_len);
1129 }
1130
Nathan Scotteedb5532005-09-02 16:39:56 +10001131 if (--bvec >= bio->bi_io_vec)
1132 prefetchw(&bvec->bv_page->flags);
1133
Nathan Scottce8e9222006-01-11 15:39:08 +11001134 if (_xfs_buf_iolocked(bp)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001135 unlock_page(page);
1136 }
Nathan Scotteedb5532005-09-02 16:39:56 +10001137 } while (bvec >= bio->bi_io_vec);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001138
Nathan Scottce8e9222006-01-11 15:39:08 +11001139 _xfs_buf_ioend(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001140 bio_put(bio);
1141 return 0;
1142}
1143
1144STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001145_xfs_buf_ioapply(
1146 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001147{
1148 int i, rw, map_i, total_nr_pages, nr_pages;
1149 struct bio *bio;
Nathan Scottce8e9222006-01-11 15:39:08 +11001150 int offset = bp->b_offset;
1151 int size = bp->b_count_desired;
1152 sector_t sector = bp->b_bn;
1153 unsigned int blocksize = bp->b_target->bt_bsize;
1154 int locking = _xfs_buf_iolocked(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001155
Nathan Scottce8e9222006-01-11 15:39:08 +11001156 total_nr_pages = bp->b_page_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001157 map_i = 0;
1158
Nathan Scottce8e9222006-01-11 15:39:08 +11001159 if (bp->b_flags & _XBF_RUN_QUEUES) {
1160 bp->b_flags &= ~_XBF_RUN_QUEUES;
1161 rw = (bp->b_flags & XBF_READ) ? READ_SYNC : WRITE_SYNC;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001162 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +11001163 rw = (bp->b_flags & XBF_READ) ? READ : WRITE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001164 }
1165
Nathan Scottce8e9222006-01-11 15:39:08 +11001166 if (bp->b_flags & XBF_ORDERED) {
1167 ASSERT(!(bp->b_flags & XBF_READ));
Christoph Hellwigf538d4d2005-11-02 10:26:59 +11001168 rw = WRITE_BARRIER;
1169 }
1170
Nathan Scottce8e9222006-01-11 15:39:08 +11001171 /* Special code path for reading a sub page size buffer in --
Linus Torvalds1da177e2005-04-16 15:20:36 -07001172 * we populate up the whole page, and hence the other metadata
1173 * in the same page. This optimization is only valid when the
Nathan Scottce8e9222006-01-11 15:39:08 +11001174 * filesystem block size is not smaller than the page size.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001175 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001176 if ((bp->b_buffer_length < PAGE_CACHE_SIZE) &&
1177 (bp->b_flags & XBF_READ) && locking &&
1178 (blocksize >= PAGE_CACHE_SIZE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001179 bio = bio_alloc(GFP_NOIO, 1);
1180
Nathan Scottce8e9222006-01-11 15:39:08 +11001181 bio->bi_bdev = bp->b_target->bt_bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001182 bio->bi_sector = sector - (offset >> BBSHIFT);
Nathan Scottce8e9222006-01-11 15:39:08 +11001183 bio->bi_end_io = xfs_buf_bio_end_io;
1184 bio->bi_private = bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001185
Nathan Scottce8e9222006-01-11 15:39:08 +11001186 bio_add_page(bio, bp->b_pages[0], PAGE_CACHE_SIZE, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001187 size = 0;
1188
Nathan Scottce8e9222006-01-11 15:39:08 +11001189 atomic_inc(&bp->b_io_remaining);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001190
1191 goto submit_io;
1192 }
1193
1194 /* Lock down the pages which we need to for the request */
Nathan Scottce8e9222006-01-11 15:39:08 +11001195 if (locking && (bp->b_flags & XBF_WRITE) && (bp->b_locked == 0)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001196 for (i = 0; size; i++) {
1197 int nbytes = PAGE_CACHE_SIZE - offset;
Nathan Scottce8e9222006-01-11 15:39:08 +11001198 struct page *page = bp->b_pages[i];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001199
1200 if (nbytes > size)
1201 nbytes = size;
1202
1203 lock_page(page);
1204
1205 size -= nbytes;
1206 offset = 0;
1207 }
Nathan Scottce8e9222006-01-11 15:39:08 +11001208 offset = bp->b_offset;
1209 size = bp->b_count_desired;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001210 }
1211
1212next_chunk:
Nathan Scottce8e9222006-01-11 15:39:08 +11001213 atomic_inc(&bp->b_io_remaining);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001214 nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1215 if (nr_pages > total_nr_pages)
1216 nr_pages = total_nr_pages;
1217
1218 bio = bio_alloc(GFP_NOIO, nr_pages);
Nathan Scottce8e9222006-01-11 15:39:08 +11001219 bio->bi_bdev = bp->b_target->bt_bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001220 bio->bi_sector = sector;
Nathan Scottce8e9222006-01-11 15:39:08 +11001221 bio->bi_end_io = xfs_buf_bio_end_io;
1222 bio->bi_private = bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001223
1224 for (; size && nr_pages; nr_pages--, map_i++) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001225 int rbytes, nbytes = PAGE_CACHE_SIZE - offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001226
1227 if (nbytes > size)
1228 nbytes = size;
1229
Nathan Scottce8e9222006-01-11 15:39:08 +11001230 rbytes = bio_add_page(bio, bp->b_pages[map_i], nbytes, offset);
1231 if (rbytes < nbytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001232 break;
1233
1234 offset = 0;
1235 sector += nbytes >> BBSHIFT;
1236 size -= nbytes;
1237 total_nr_pages--;
1238 }
1239
1240submit_io:
1241 if (likely(bio->bi_size)) {
1242 submit_bio(rw, bio);
1243 if (size)
1244 goto next_chunk;
1245 } else {
1246 bio_put(bio);
Nathan Scottce8e9222006-01-11 15:39:08 +11001247 xfs_buf_ioerror(bp, EIO);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001248 }
1249}
1250
Linus Torvalds1da177e2005-04-16 15:20:36 -07001251int
Nathan Scottce8e9222006-01-11 15:39:08 +11001252xfs_buf_iorequest(
1253 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001254{
Nathan Scottce8e9222006-01-11 15:39:08 +11001255 XB_TRACE(bp, "iorequest", 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001256
Nathan Scottce8e9222006-01-11 15:39:08 +11001257 if (bp->b_flags & XBF_DELWRI) {
1258 xfs_buf_delwri_queue(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001259 return 0;
1260 }
1261
Nathan Scottce8e9222006-01-11 15:39:08 +11001262 if (bp->b_flags & XBF_WRITE) {
1263 xfs_buf_wait_unpin(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001264 }
1265
Nathan Scottce8e9222006-01-11 15:39:08 +11001266 xfs_buf_hold(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001267
1268 /* Set the count to 1 initially, this will stop an I/O
1269 * completion callout which happens before we have started
Nathan Scottce8e9222006-01-11 15:39:08 +11001270 * all the I/O from calling xfs_buf_ioend too early.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001271 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001272 atomic_set(&bp->b_io_remaining, 1);
1273 _xfs_buf_ioapply(bp);
1274 _xfs_buf_ioend(bp, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001275
Nathan Scottce8e9222006-01-11 15:39:08 +11001276 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001277 return 0;
1278}
1279
1280/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001281 * Waits for I/O to complete on the buffer supplied.
1282 * It returns immediately if no I/O is pending.
1283 * It returns the I/O error code, if any, or 0 if there was no error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001284 */
1285int
Nathan Scottce8e9222006-01-11 15:39:08 +11001286xfs_buf_iowait(
1287 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001288{
Nathan Scottce8e9222006-01-11 15:39:08 +11001289 XB_TRACE(bp, "iowait", 0);
1290 if (atomic_read(&bp->b_io_remaining))
1291 blk_run_address_space(bp->b_target->bt_mapping);
1292 down(&bp->b_iodonesema);
1293 XB_TRACE(bp, "iowaited", (long)bp->b_error);
1294 return bp->b_error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001295}
1296
Nathan Scottce8e9222006-01-11 15:39:08 +11001297xfs_caddr_t
1298xfs_buf_offset(
1299 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001300 size_t offset)
1301{
1302 struct page *page;
1303
Nathan Scottce8e9222006-01-11 15:39:08 +11001304 if (bp->b_flags & XBF_MAPPED)
1305 return XFS_BUF_PTR(bp) + offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001306
Nathan Scottce8e9222006-01-11 15:39:08 +11001307 offset += bp->b_offset;
1308 page = bp->b_pages[offset >> PAGE_CACHE_SHIFT];
1309 return (xfs_caddr_t)page_address(page) + (offset & (PAGE_CACHE_SIZE-1));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001310}
1311
1312/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001313 * Move data into or out of a buffer.
1314 */
1315void
Nathan Scottce8e9222006-01-11 15:39:08 +11001316xfs_buf_iomove(
1317 xfs_buf_t *bp, /* buffer to process */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001318 size_t boff, /* starting buffer offset */
1319 size_t bsize, /* length to copy */
1320 caddr_t data, /* data address */
Nathan Scottce8e9222006-01-11 15:39:08 +11001321 xfs_buf_rw_t mode) /* read/write/zero flag */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001322{
1323 size_t bend, cpoff, csize;
1324 struct page *page;
1325
1326 bend = boff + bsize;
1327 while (boff < bend) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001328 page = bp->b_pages[xfs_buf_btoct(boff + bp->b_offset)];
1329 cpoff = xfs_buf_poff(boff + bp->b_offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001330 csize = min_t(size_t,
Nathan Scottce8e9222006-01-11 15:39:08 +11001331 PAGE_CACHE_SIZE-cpoff, bp->b_count_desired-boff);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001332
1333 ASSERT(((csize + cpoff) <= PAGE_CACHE_SIZE));
1334
1335 switch (mode) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001336 case XBRW_ZERO:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001337 memset(page_address(page) + cpoff, 0, csize);
1338 break;
Nathan Scottce8e9222006-01-11 15:39:08 +11001339 case XBRW_READ:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001340 memcpy(data, page_address(page) + cpoff, csize);
1341 break;
Nathan Scottce8e9222006-01-11 15:39:08 +11001342 case XBRW_WRITE:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001343 memcpy(page_address(page) + cpoff, data, csize);
1344 }
1345
1346 boff += csize;
1347 data += csize;
1348 }
1349}
1350
1351/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001352 * Handling of buffer targets (buftargs).
Linus Torvalds1da177e2005-04-16 15:20:36 -07001353 */
1354
1355/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001356 * Wait for any bufs with callbacks that have been submitted but
1357 * have not yet returned... walk the hash list for the target.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001358 */
1359void
1360xfs_wait_buftarg(
1361 xfs_buftarg_t *btp)
1362{
1363 xfs_buf_t *bp, *n;
1364 xfs_bufhash_t *hash;
1365 uint i;
1366
1367 for (i = 0; i < (1 << btp->bt_hashshift); i++) {
1368 hash = &btp->bt_hash[i];
1369again:
1370 spin_lock(&hash->bh_lock);
Nathan Scottce8e9222006-01-11 15:39:08 +11001371 list_for_each_entry_safe(bp, n, &hash->bh_list, b_hash_list) {
1372 ASSERT(btp == bp->b_target);
1373 if (!(bp->b_flags & XBF_FS_MANAGED)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001374 spin_unlock(&hash->bh_lock);
David Chinner2f926582005-09-05 08:33:35 +10001375 /*
1376 * Catch superblock reference count leaks
1377 * immediately
1378 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001379 BUG_ON(bp->b_bn == 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001380 delay(100);
1381 goto again;
1382 }
1383 }
1384 spin_unlock(&hash->bh_lock);
1385 }
1386}
1387
1388/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001389 * Allocate buffer hash table for a given target.
1390 * For devices containing metadata (i.e. not the log/realtime devices)
1391 * we need to allocate a much larger hash table.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001392 */
1393STATIC void
1394xfs_alloc_bufhash(
1395 xfs_buftarg_t *btp,
1396 int external)
1397{
1398 unsigned int i;
1399
1400 btp->bt_hashshift = external ? 3 : 8; /* 8 or 256 buckets */
1401 btp->bt_hashmask = (1 << btp->bt_hashshift) - 1;
1402 btp->bt_hash = kmem_zalloc((1 << btp->bt_hashshift) *
1403 sizeof(xfs_bufhash_t), KM_SLEEP);
1404 for (i = 0; i < (1 << btp->bt_hashshift); i++) {
1405 spin_lock_init(&btp->bt_hash[i].bh_lock);
1406 INIT_LIST_HEAD(&btp->bt_hash[i].bh_list);
1407 }
1408}
1409
1410STATIC void
1411xfs_free_bufhash(
1412 xfs_buftarg_t *btp)
1413{
Nathan Scottce8e9222006-01-11 15:39:08 +11001414 kmem_free(btp->bt_hash, (1<<btp->bt_hashshift) * sizeof(xfs_bufhash_t));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001415 btp->bt_hash = NULL;
1416}
1417
David Chinnera6867a62006-01-11 15:37:58 +11001418/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001419 * buftarg list for delwrite queue processing
David Chinnera6867a62006-01-11 15:37:58 +11001420 */
1421STATIC LIST_HEAD(xfs_buftarg_list);
1422STATIC DEFINE_SPINLOCK(xfs_buftarg_lock);
1423
1424STATIC void
1425xfs_register_buftarg(
1426 xfs_buftarg_t *btp)
1427{
1428 spin_lock(&xfs_buftarg_lock);
1429 list_add(&btp->bt_list, &xfs_buftarg_list);
1430 spin_unlock(&xfs_buftarg_lock);
1431}
1432
1433STATIC void
1434xfs_unregister_buftarg(
1435 xfs_buftarg_t *btp)
1436{
1437 spin_lock(&xfs_buftarg_lock);
1438 list_del(&btp->bt_list);
1439 spin_unlock(&xfs_buftarg_lock);
1440}
1441
Linus Torvalds1da177e2005-04-16 15:20:36 -07001442void
1443xfs_free_buftarg(
1444 xfs_buftarg_t *btp,
1445 int external)
1446{
1447 xfs_flush_buftarg(btp, 1);
1448 if (external)
Nathan Scottce8e9222006-01-11 15:39:08 +11001449 xfs_blkdev_put(btp->bt_bdev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001450 xfs_free_bufhash(btp);
Nathan Scottce8e9222006-01-11 15:39:08 +11001451 iput(btp->bt_mapping->host);
David Chinnera6867a62006-01-11 15:37:58 +11001452
Nathan Scottce8e9222006-01-11 15:39:08 +11001453 /* Unregister the buftarg first so that we don't get a
1454 * wakeup finding a non-existent task
1455 */
David Chinnera6867a62006-01-11 15:37:58 +11001456 xfs_unregister_buftarg(btp);
1457 kthread_stop(btp->bt_task);
1458
Linus Torvalds1da177e2005-04-16 15:20:36 -07001459 kmem_free(btp, sizeof(*btp));
1460}
1461
Linus Torvalds1da177e2005-04-16 15:20:36 -07001462STATIC int
1463xfs_setsize_buftarg_flags(
1464 xfs_buftarg_t *btp,
1465 unsigned int blocksize,
1466 unsigned int sectorsize,
1467 int verbose)
1468{
Nathan Scottce8e9222006-01-11 15:39:08 +11001469 btp->bt_bsize = blocksize;
1470 btp->bt_sshift = ffs(sectorsize) - 1;
1471 btp->bt_smask = sectorsize - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001472
Nathan Scottce8e9222006-01-11 15:39:08 +11001473 if (set_blocksize(btp->bt_bdev, sectorsize)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001474 printk(KERN_WARNING
1475 "XFS: Cannot set_blocksize to %u on device %s\n",
1476 sectorsize, XFS_BUFTARG_NAME(btp));
1477 return EINVAL;
1478 }
1479
1480 if (verbose &&
1481 (PAGE_CACHE_SIZE / BITS_PER_LONG) > sectorsize) {
1482 printk(KERN_WARNING
1483 "XFS: %u byte sectors in use on device %s. "
1484 "This is suboptimal; %u or greater is ideal.\n",
1485 sectorsize, XFS_BUFTARG_NAME(btp),
1486 (unsigned int)PAGE_CACHE_SIZE / BITS_PER_LONG);
1487 }
1488
1489 return 0;
1490}
1491
1492/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001493 * When allocating the initial buffer target we have not yet
1494 * read in the superblock, so don't know what sized sectors
1495 * are being used is at this early stage. Play safe.
1496 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001497STATIC int
1498xfs_setsize_buftarg_early(
1499 xfs_buftarg_t *btp,
1500 struct block_device *bdev)
1501{
1502 return xfs_setsize_buftarg_flags(btp,
1503 PAGE_CACHE_SIZE, bdev_hardsect_size(bdev), 0);
1504}
1505
1506int
1507xfs_setsize_buftarg(
1508 xfs_buftarg_t *btp,
1509 unsigned int blocksize,
1510 unsigned int sectorsize)
1511{
1512 return xfs_setsize_buftarg_flags(btp, blocksize, sectorsize, 1);
1513}
1514
1515STATIC int
1516xfs_mapping_buftarg(
1517 xfs_buftarg_t *btp,
1518 struct block_device *bdev)
1519{
1520 struct backing_dev_info *bdi;
1521 struct inode *inode;
1522 struct address_space *mapping;
1523 static struct address_space_operations mapping_aops = {
1524 .sync_page = block_sync_page,
Christoph Lametere965f962006-02-01 03:05:41 -08001525 .migratepage = fail_migrate_page,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001526 };
1527
1528 inode = new_inode(bdev->bd_inode->i_sb);
1529 if (!inode) {
1530 printk(KERN_WARNING
1531 "XFS: Cannot allocate mapping inode for device %s\n",
1532 XFS_BUFTARG_NAME(btp));
1533 return ENOMEM;
1534 }
1535 inode->i_mode = S_IFBLK;
1536 inode->i_bdev = bdev;
1537 inode->i_rdev = bdev->bd_dev;
1538 bdi = blk_get_backing_dev_info(bdev);
1539 if (!bdi)
1540 bdi = &default_backing_dev_info;
1541 mapping = &inode->i_data;
1542 mapping->a_ops = &mapping_aops;
1543 mapping->backing_dev_info = bdi;
1544 mapping_set_gfp_mask(mapping, GFP_NOFS);
Nathan Scottce8e9222006-01-11 15:39:08 +11001545 btp->bt_mapping = mapping;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001546 return 0;
1547}
1548
David Chinnera6867a62006-01-11 15:37:58 +11001549STATIC int
1550xfs_alloc_delwrite_queue(
1551 xfs_buftarg_t *btp)
1552{
1553 int error = 0;
1554
1555 INIT_LIST_HEAD(&btp->bt_list);
1556 INIT_LIST_HEAD(&btp->bt_delwrite_queue);
1557 spinlock_init(&btp->bt_delwrite_lock, "delwri_lock");
1558 btp->bt_flags = 0;
1559 btp->bt_task = kthread_run(xfsbufd, btp, "xfsbufd");
1560 if (IS_ERR(btp->bt_task)) {
1561 error = PTR_ERR(btp->bt_task);
1562 goto out_error;
1563 }
1564 xfs_register_buftarg(btp);
1565out_error:
1566 return error;
1567}
1568
Linus Torvalds1da177e2005-04-16 15:20:36 -07001569xfs_buftarg_t *
1570xfs_alloc_buftarg(
1571 struct block_device *bdev,
1572 int external)
1573{
1574 xfs_buftarg_t *btp;
1575
1576 btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
1577
Nathan Scottce8e9222006-01-11 15:39:08 +11001578 btp->bt_dev = bdev->bd_dev;
1579 btp->bt_bdev = bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001580 if (xfs_setsize_buftarg_early(btp, bdev))
1581 goto error;
1582 if (xfs_mapping_buftarg(btp, bdev))
1583 goto error;
David Chinnera6867a62006-01-11 15:37:58 +11001584 if (xfs_alloc_delwrite_queue(btp))
1585 goto error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001586 xfs_alloc_bufhash(btp, external);
1587 return btp;
1588
1589error:
1590 kmem_free(btp, sizeof(*btp));
1591 return NULL;
1592}
1593
1594
1595/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001596 * Delayed write buffer handling
Linus Torvalds1da177e2005-04-16 15:20:36 -07001597 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001598STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001599xfs_buf_delwri_queue(
1600 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001601 int unlock)
1602{
Nathan Scottce8e9222006-01-11 15:39:08 +11001603 struct list_head *dwq = &bp->b_target->bt_delwrite_queue;
1604 spinlock_t *dwlk = &bp->b_target->bt_delwrite_lock;
David Chinnera6867a62006-01-11 15:37:58 +11001605
Nathan Scottce8e9222006-01-11 15:39:08 +11001606 XB_TRACE(bp, "delwri_q", (long)unlock);
1607 ASSERT((bp->b_flags&(XBF_DELWRI|XBF_ASYNC)) == (XBF_DELWRI|XBF_ASYNC));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001608
David Chinnera6867a62006-01-11 15:37:58 +11001609 spin_lock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001610 /* If already in the queue, dequeue and place at tail */
Nathan Scottce8e9222006-01-11 15:39:08 +11001611 if (!list_empty(&bp->b_list)) {
1612 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1613 if (unlock)
1614 atomic_dec(&bp->b_hold);
1615 list_del(&bp->b_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001616 }
1617
Nathan Scottce8e9222006-01-11 15:39:08 +11001618 bp->b_flags |= _XBF_DELWRI_Q;
1619 list_add_tail(&bp->b_list, dwq);
1620 bp->b_queuetime = jiffies;
David Chinnera6867a62006-01-11 15:37:58 +11001621 spin_unlock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001622
1623 if (unlock)
Nathan Scottce8e9222006-01-11 15:39:08 +11001624 xfs_buf_unlock(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001625}
1626
1627void
Nathan Scottce8e9222006-01-11 15:39:08 +11001628xfs_buf_delwri_dequeue(
1629 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001630{
Nathan Scottce8e9222006-01-11 15:39:08 +11001631 spinlock_t *dwlk = &bp->b_target->bt_delwrite_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001632 int dequeued = 0;
1633
David Chinnera6867a62006-01-11 15:37:58 +11001634 spin_lock(dwlk);
Nathan Scottce8e9222006-01-11 15:39:08 +11001635 if ((bp->b_flags & XBF_DELWRI) && !list_empty(&bp->b_list)) {
1636 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1637 list_del_init(&bp->b_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001638 dequeued = 1;
1639 }
Nathan Scottce8e9222006-01-11 15:39:08 +11001640 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
David Chinnera6867a62006-01-11 15:37:58 +11001641 spin_unlock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001642
1643 if (dequeued)
Nathan Scottce8e9222006-01-11 15:39:08 +11001644 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001645
Nathan Scottce8e9222006-01-11 15:39:08 +11001646 XB_TRACE(bp, "delwri_dq", (long)dequeued);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001647}
1648
1649STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001650xfs_buf_runall_queues(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001651 struct workqueue_struct *queue)
1652{
1653 flush_workqueue(queue);
1654}
1655
Linus Torvalds1da177e2005-04-16 15:20:36 -07001656STATIC int
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001657xfsbufd_wakeup(
Nathan Scott15c84a42005-11-04 10:51:01 +11001658 int priority,
1659 gfp_t mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001660{
Christoph Hellwigda7f93e2006-01-11 20:49:57 +11001661 xfs_buftarg_t *btp;
David Chinnera6867a62006-01-11 15:37:58 +11001662
1663 spin_lock(&xfs_buftarg_lock);
Christoph Hellwigda7f93e2006-01-11 20:49:57 +11001664 list_for_each_entry(btp, &xfs_buftarg_list, bt_list) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001665 if (test_bit(XBT_FORCE_SLEEP, &btp->bt_flags))
David Chinnera6867a62006-01-11 15:37:58 +11001666 continue;
Nathan Scottce8e9222006-01-11 15:39:08 +11001667 set_bit(XBT_FORCE_FLUSH, &btp->bt_flags);
David Chinnera6867a62006-01-11 15:37:58 +11001668 wake_up_process(btp->bt_task);
1669 }
1670 spin_unlock(&xfs_buftarg_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001671 return 0;
1672}
1673
1674STATIC int
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001675xfsbufd(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001676 void *data)
1677{
1678 struct list_head tmp;
1679 unsigned long age;
David Chinnera6867a62006-01-11 15:37:58 +11001680 xfs_buftarg_t *target = (xfs_buftarg_t *)data;
Nathan Scottce8e9222006-01-11 15:39:08 +11001681 xfs_buf_t *bp, *n;
David Chinnera6867a62006-01-11 15:37:58 +11001682 struct list_head *dwq = &target->bt_delwrite_queue;
1683 spinlock_t *dwlk = &target->bt_delwrite_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001684
Linus Torvalds1da177e2005-04-16 15:20:36 -07001685 current->flags |= PF_MEMALLOC;
1686
Linus Torvalds1da177e2005-04-16 15:20:36 -07001687 INIT_LIST_HEAD(&tmp);
1688 do {
Christoph Lameter3e1d1d22005-06-24 23:13:50 -07001689 if (unlikely(freezing(current))) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001690 set_bit(XBT_FORCE_SLEEP, &target->bt_flags);
Christoph Lameter3e1d1d22005-06-24 23:13:50 -07001691 refrigerator();
Nathan Scottabd0cf72005-05-05 13:30:13 -07001692 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +11001693 clear_bit(XBT_FORCE_SLEEP, &target->bt_flags);
Nathan Scottabd0cf72005-05-05 13:30:13 -07001694 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001695
Nathan Scott15c84a42005-11-04 10:51:01 +11001696 schedule_timeout_interruptible(
1697 xfs_buf_timer_centisecs * msecs_to_jiffies(10));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001698
Nishanth Aravamudan041e0e32005-09-10 00:27:23 -07001699 age = xfs_buf_age_centisecs * msecs_to_jiffies(10);
David Chinnera6867a62006-01-11 15:37:58 +11001700 spin_lock(dwlk);
Nathan Scottce8e9222006-01-11 15:39:08 +11001701 list_for_each_entry_safe(bp, n, dwq, b_list) {
1702 XB_TRACE(bp, "walkq1", (long)xfs_buf_ispin(bp));
1703 ASSERT(bp->b_flags & XBF_DELWRI);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001704
Nathan Scottce8e9222006-01-11 15:39:08 +11001705 if (!xfs_buf_ispin(bp) && !xfs_buf_cond_lock(bp)) {
1706 if (!test_bit(XBT_FORCE_FLUSH,
David Chinnera6867a62006-01-11 15:37:58 +11001707 &target->bt_flags) &&
Linus Torvalds1da177e2005-04-16 15:20:36 -07001708 time_before(jiffies,
Nathan Scottce8e9222006-01-11 15:39:08 +11001709 bp->b_queuetime + age)) {
1710 xfs_buf_unlock(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001711 break;
1712 }
1713
Nathan Scottce8e9222006-01-11 15:39:08 +11001714 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
1715 bp->b_flags |= XBF_WRITE;
1716 list_move(&bp->b_list, &tmp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001717 }
1718 }
David Chinnera6867a62006-01-11 15:37:58 +11001719 spin_unlock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001720
1721 while (!list_empty(&tmp)) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001722 bp = list_entry(tmp.next, xfs_buf_t, b_list);
1723 ASSERT(target == bp->b_target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001724
Nathan Scottce8e9222006-01-11 15:39:08 +11001725 list_del_init(&bp->b_list);
1726 xfs_buf_iostrategy(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001727
Nathan Scottce8e9222006-01-11 15:39:08 +11001728 blk_run_address_space(target->bt_mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001729 }
1730
1731 if (as_list_len > 0)
1732 purge_addresses();
1733
Nathan Scottce8e9222006-01-11 15:39:08 +11001734 clear_bit(XBT_FORCE_FLUSH, &target->bt_flags);
Christoph Hellwig4df08c52005-09-05 08:34:18 +10001735 } while (!kthread_should_stop());
Linus Torvalds1da177e2005-04-16 15:20:36 -07001736
Christoph Hellwig4df08c52005-09-05 08:34:18 +10001737 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001738}
1739
1740/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001741 * Go through all incore buffers, and release buffers if they belong to
1742 * the given device. This is used in filesystem error handling to
1743 * preserve the consistency of its metadata.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001744 */
1745int
1746xfs_flush_buftarg(
1747 xfs_buftarg_t *target,
1748 int wait)
1749{
1750 struct list_head tmp;
Nathan Scottce8e9222006-01-11 15:39:08 +11001751 xfs_buf_t *bp, *n;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001752 int pincount = 0;
David Chinnera6867a62006-01-11 15:37:58 +11001753 struct list_head *dwq = &target->bt_delwrite_queue;
1754 spinlock_t *dwlk = &target->bt_delwrite_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001755
Nathan Scottce8e9222006-01-11 15:39:08 +11001756 xfs_buf_runall_queues(xfsdatad_workqueue);
1757 xfs_buf_runall_queues(xfslogd_workqueue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001758
1759 INIT_LIST_HEAD(&tmp);
David Chinnera6867a62006-01-11 15:37:58 +11001760 spin_lock(dwlk);
Nathan Scottce8e9222006-01-11 15:39:08 +11001761 list_for_each_entry_safe(bp, n, dwq, b_list) {
1762 ASSERT(bp->b_target == target);
1763 ASSERT(bp->b_flags & (XBF_DELWRI | _XBF_DELWRI_Q));
1764 XB_TRACE(bp, "walkq2", (long)xfs_buf_ispin(bp));
1765 if (xfs_buf_ispin(bp)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001766 pincount++;
1767 continue;
1768 }
1769
Nathan Scottce8e9222006-01-11 15:39:08 +11001770 list_move(&bp->b_list, &tmp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001771 }
David Chinnera6867a62006-01-11 15:37:58 +11001772 spin_unlock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001773
1774 /*
1775 * Dropped the delayed write list lock, now walk the temporary list
1776 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001777 list_for_each_entry_safe(bp, n, &tmp, b_list) {
1778 xfs_buf_lock(bp);
1779 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
1780 bp->b_flags |= XBF_WRITE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001781 if (wait)
Nathan Scottce8e9222006-01-11 15:39:08 +11001782 bp->b_flags &= ~XBF_ASYNC;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001783 else
Nathan Scottce8e9222006-01-11 15:39:08 +11001784 list_del_init(&bp->b_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001785
Nathan Scottce8e9222006-01-11 15:39:08 +11001786 xfs_buf_iostrategy(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001787 }
1788
1789 /*
1790 * Remaining list items must be flushed before returning
1791 */
1792 while (!list_empty(&tmp)) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001793 bp = list_entry(tmp.next, xfs_buf_t, b_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001794
Nathan Scottce8e9222006-01-11 15:39:08 +11001795 list_del_init(&bp->b_list);
1796 xfs_iowait(bp);
1797 xfs_buf_relse(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001798 }
1799
1800 if (wait)
Nathan Scottce8e9222006-01-11 15:39:08 +11001801 blk_run_address_space(target->bt_mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001802
1803 return pincount;
1804}
1805
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001806int __init
Nathan Scottce8e9222006-01-11 15:39:08 +11001807xfs_buf_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001808{
Nathan Scottce8e9222006-01-11 15:39:08 +11001809#ifdef XFS_BUF_TRACE
1810 xfs_buf_trace_buf = ktrace_alloc(XFS_BUF_TRACE_SIZE, KM_SLEEP);
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001811#endif
1812
Nathan Scott87582802006-03-14 13:18:19 +11001813 xfs_buf_zone = kmem_zone_init_flags(sizeof(xfs_buf_t), "xfs_buf",
1814 KM_ZONE_HWALIGN, NULL);
Nathan Scottce8e9222006-01-11 15:39:08 +11001815 if (!xfs_buf_zone)
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001816 goto out_free_trace_buf;
1817
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001818 xfslogd_workqueue = create_workqueue("xfslogd");
1819 if (!xfslogd_workqueue)
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001820 goto out_free_buf_zone;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001821
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001822 xfsdatad_workqueue = create_workqueue("xfsdatad");
1823 if (!xfsdatad_workqueue)
1824 goto out_destroy_xfslogd_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001825
Nathan Scottce8e9222006-01-11 15:39:08 +11001826 xfs_buf_shake = kmem_shake_register(xfsbufd_wakeup);
1827 if (!xfs_buf_shake)
David Chinnera6867a62006-01-11 15:37:58 +11001828 goto out_destroy_xfsdatad_workqueue;
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001829
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001830 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001831
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001832 out_destroy_xfsdatad_workqueue:
1833 destroy_workqueue(xfsdatad_workqueue);
1834 out_destroy_xfslogd_workqueue:
1835 destroy_workqueue(xfslogd_workqueue);
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001836 out_free_buf_zone:
Nathan Scottce8e9222006-01-11 15:39:08 +11001837 kmem_zone_destroy(xfs_buf_zone);
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001838 out_free_trace_buf:
Nathan Scottce8e9222006-01-11 15:39:08 +11001839#ifdef XFS_BUF_TRACE
1840 ktrace_free(xfs_buf_trace_buf);
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001841#endif
Nathan Scott87582802006-03-14 13:18:19 +11001842 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001843}
1844
Linus Torvalds1da177e2005-04-16 15:20:36 -07001845void
Nathan Scottce8e9222006-01-11 15:39:08 +11001846xfs_buf_terminate(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001847{
Nathan Scottce8e9222006-01-11 15:39:08 +11001848 kmem_shake_deregister(xfs_buf_shake);
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001849 destroy_workqueue(xfsdatad_workqueue);
1850 destroy_workqueue(xfslogd_workqueue);
Nathan Scottce8e9222006-01-11 15:39:08 +11001851 kmem_zone_destroy(xfs_buf_zone);
1852#ifdef XFS_BUF_TRACE
1853 ktrace_free(xfs_buf_trace_buf);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001854#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001855}