blob: a49dd8d4b06974c9a9b8b20ca19bc0663dfe0471 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Nathan Scottf07c2252006-09-28 10:52:15 +10002 * Copyright (c) 2000-2006 Silicon Graphics, Inc.
Nathan Scott7b718762005-11-02 14:58:39 +11003 * All Rights Reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
Nathan Scott7b718762005-11-02 14:58:39 +11005 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License as
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 * published by the Free Software Foundation.
8 *
Nathan Scott7b718762005-11-02 14:58:39 +11009 * This program is distributed in the hope that it would be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 *
Nathan Scott7b718762005-11-02 14:58:39 +110014 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
Linus Torvalds1da177e2005-04-16 15:20:36 -070017 */
Vlad Apostolov93c189c2006-11-11 18:03:49 +110018#include "xfs.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/stddef.h>
20#include <linux/errno.h>
21#include <linux/slab.h>
22#include <linux/pagemap.h>
23#include <linux/init.h>
24#include <linux/vmalloc.h>
25#include <linux/bio.h>
26#include <linux/sysctl.h>
27#include <linux/proc_fs.h>
28#include <linux/workqueue.h>
29#include <linux/percpu.h>
30#include <linux/blkdev.h>
31#include <linux/hash.h>
Christoph Hellwig4df08c52005-09-05 08:34:18 +100032#include <linux/kthread.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080033#include <linux/migrate.h>
Andrew Morton3fcfab12006-10-19 23:28:16 -070034#include <linux/backing-dev.h>
Nigel Cunningham7dfb7102006-12-06 20:34:23 -080035#include <linux/freezer.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070036
David Chinner7989cb82007-02-10 18:34:56 +110037static kmem_zone_t *xfs_buf_zone;
David Chinnera6867a62006-01-11 15:37:58 +110038STATIC int xfsbufd(void *);
Al Viro27496a82005-10-21 03:20:48 -040039STATIC int xfsbufd_wakeup(int, gfp_t);
Nathan Scottce8e9222006-01-11 15:39:08 +110040STATIC void xfs_buf_delwri_queue(xfs_buf_t *, int);
Rusty Russell8e1f9362007-07-17 04:03:17 -070041static struct shrinker xfs_buf_shake = {
42 .shrink = xfsbufd_wakeup,
43 .seeks = DEFAULT_SEEKS,
44};
Christoph Hellwig23ea4032005-06-21 15:14:01 +100045
David Chinner7989cb82007-02-10 18:34:56 +110046static struct workqueue_struct *xfslogd_workqueue;
Christoph Hellwig0829c362005-09-02 16:58:49 +100047struct workqueue_struct *xfsdatad_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070048
Nathan Scottce8e9222006-01-11 15:39:08 +110049#ifdef XFS_BUF_TRACE
Linus Torvalds1da177e2005-04-16 15:20:36 -070050void
Nathan Scottce8e9222006-01-11 15:39:08 +110051xfs_buf_trace(
52 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -070053 char *id,
54 void *data,
55 void *ra)
56{
Nathan Scottce8e9222006-01-11 15:39:08 +110057 ktrace_enter(xfs_buf_trace_buf,
58 bp, id,
59 (void *)(unsigned long)bp->b_flags,
60 (void *)(unsigned long)bp->b_hold.counter,
61 (void *)(unsigned long)bp->b_sema.count.counter,
Linus Torvalds1da177e2005-04-16 15:20:36 -070062 (void *)current,
63 data, ra,
Nathan Scottce8e9222006-01-11 15:39:08 +110064 (void *)(unsigned long)((bp->b_file_offset>>32) & 0xffffffff),
65 (void *)(unsigned long)(bp->b_file_offset & 0xffffffff),
66 (void *)(unsigned long)bp->b_buffer_length,
Linus Torvalds1da177e2005-04-16 15:20:36 -070067 NULL, NULL, NULL, NULL, NULL);
68}
Nathan Scottce8e9222006-01-11 15:39:08 +110069ktrace_t *xfs_buf_trace_buf;
70#define XFS_BUF_TRACE_SIZE 4096
71#define XB_TRACE(bp, id, data) \
72 xfs_buf_trace(bp, id, (void *)data, (void *)__builtin_return_address(0))
Linus Torvalds1da177e2005-04-16 15:20:36 -070073#else
Nathan Scottce8e9222006-01-11 15:39:08 +110074#define XB_TRACE(bp, id, data) do { } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -070075#endif
76
Nathan Scottce8e9222006-01-11 15:39:08 +110077#ifdef XFS_BUF_LOCK_TRACKING
78# define XB_SET_OWNER(bp) ((bp)->b_last_holder = current->pid)
79# define XB_CLEAR_OWNER(bp) ((bp)->b_last_holder = -1)
80# define XB_GET_OWNER(bp) ((bp)->b_last_holder)
Linus Torvalds1da177e2005-04-16 15:20:36 -070081#else
Nathan Scottce8e9222006-01-11 15:39:08 +110082# define XB_SET_OWNER(bp) do { } while (0)
83# define XB_CLEAR_OWNER(bp) do { } while (0)
84# define XB_GET_OWNER(bp) do { } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -070085#endif
86
Nathan Scottce8e9222006-01-11 15:39:08 +110087#define xb_to_gfp(flags) \
88 ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : \
89 ((flags) & XBF_DONT_BLOCK) ? GFP_NOFS : GFP_KERNEL) | __GFP_NOWARN)
Linus Torvalds1da177e2005-04-16 15:20:36 -070090
Nathan Scottce8e9222006-01-11 15:39:08 +110091#define xb_to_km(flags) \
92 (((flags) & XBF_DONT_BLOCK) ? KM_NOFS : KM_SLEEP)
Linus Torvalds1da177e2005-04-16 15:20:36 -070093
Nathan Scottce8e9222006-01-11 15:39:08 +110094#define xfs_buf_allocate(flags) \
95 kmem_zone_alloc(xfs_buf_zone, xb_to_km(flags))
96#define xfs_buf_deallocate(bp) \
97 kmem_zone_free(xfs_buf_zone, (bp));
Linus Torvalds1da177e2005-04-16 15:20:36 -070098
99/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100100 * Page Region interfaces.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101 *
Nathan Scottce8e9222006-01-11 15:39:08 +1100102 * For pages in filesystems where the blocksize is smaller than the
103 * pagesize, we use the page->private field (long) to hold a bitmap
104 * of uptodate regions within the page.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105 *
Nathan Scottce8e9222006-01-11 15:39:08 +1100106 * Each such region is "bytes per page / bits per long" bytes long.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107 *
Nathan Scottce8e9222006-01-11 15:39:08 +1100108 * NBPPR == number-of-bytes-per-page-region
109 * BTOPR == bytes-to-page-region (rounded up)
110 * BTOPRT == bytes-to-page-region-truncated (rounded down)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111 */
112#if (BITS_PER_LONG == 32)
113#define PRSHIFT (PAGE_CACHE_SHIFT - 5) /* (32 == 1<<5) */
114#elif (BITS_PER_LONG == 64)
115#define PRSHIFT (PAGE_CACHE_SHIFT - 6) /* (64 == 1<<6) */
116#else
117#error BITS_PER_LONG must be 32 or 64
118#endif
119#define NBPPR (PAGE_CACHE_SIZE/BITS_PER_LONG)
120#define BTOPR(b) (((unsigned int)(b) + (NBPPR - 1)) >> PRSHIFT)
121#define BTOPRT(b) (((unsigned int)(b) >> PRSHIFT))
122
123STATIC unsigned long
124page_region_mask(
125 size_t offset,
126 size_t length)
127{
128 unsigned long mask;
129 int first, final;
130
131 first = BTOPR(offset);
132 final = BTOPRT(offset + length - 1);
133 first = min(first, final);
134
135 mask = ~0UL;
136 mask <<= BITS_PER_LONG - (final - first);
137 mask >>= BITS_PER_LONG - (final);
138
139 ASSERT(offset + length <= PAGE_CACHE_SIZE);
140 ASSERT((final - first) < BITS_PER_LONG && (final - first) >= 0);
141
142 return mask;
143}
144
David Chinner7989cb82007-02-10 18:34:56 +1100145STATIC_INLINE void
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146set_page_region(
147 struct page *page,
148 size_t offset,
149 size_t length)
150{
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700151 set_page_private(page,
152 page_private(page) | page_region_mask(offset, length));
153 if (page_private(page) == ~0UL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700154 SetPageUptodate(page);
155}
156
David Chinner7989cb82007-02-10 18:34:56 +1100157STATIC_INLINE int
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158test_page_region(
159 struct page *page,
160 size_t offset,
161 size_t length)
162{
163 unsigned long mask = page_region_mask(offset, length);
164
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700165 return (mask && (page_private(page) & mask) == mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166}
167
168/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100169 * Mapping of multi-page buffers into contiguous virtual space
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170 */
171
172typedef struct a_list {
173 void *vm_addr;
174 struct a_list *next;
175} a_list_t;
176
David Chinner7989cb82007-02-10 18:34:56 +1100177static a_list_t *as_free_head;
178static int as_list_len;
179static DEFINE_SPINLOCK(as_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700180
181/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100182 * Try to batch vunmaps because they are costly.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183 */
184STATIC void
185free_address(
186 void *addr)
187{
188 a_list_t *aentry;
189
Jeremy Fitzhardinge7f015072007-10-17 13:55:03 +1000190#ifdef CONFIG_XEN
191 /*
192 * Xen needs to be able to make sure it can get an exclusive
193 * RO mapping of pages it wants to turn into a pagetable. If
194 * a newly allocated page is also still being vmap()ed by xfs,
195 * it will cause pagetable construction to fail. This is a
196 * quick workaround to always eagerly unmap pages so that Xen
197 * is happy.
198 */
199 vunmap(addr);
200 return;
201#endif
202
Jeff Dike7b04d712006-04-10 22:53:27 -0700203 aentry = kmalloc(sizeof(a_list_t), GFP_NOWAIT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204 if (likely(aentry)) {
205 spin_lock(&as_lock);
206 aentry->next = as_free_head;
207 aentry->vm_addr = addr;
208 as_free_head = aentry;
209 as_list_len++;
210 spin_unlock(&as_lock);
211 } else {
212 vunmap(addr);
213 }
214}
215
216STATIC void
217purge_addresses(void)
218{
219 a_list_t *aentry, *old;
220
221 if (as_free_head == NULL)
222 return;
223
224 spin_lock(&as_lock);
225 aentry = as_free_head;
226 as_free_head = NULL;
227 as_list_len = 0;
228 spin_unlock(&as_lock);
229
230 while ((old = aentry) != NULL) {
231 vunmap(aentry->vm_addr);
232 aentry = aentry->next;
233 kfree(old);
234 }
235}
236
237/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100238 * Internal xfs_buf_t object manipulation
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239 */
240
241STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +1100242_xfs_buf_initialize(
243 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244 xfs_buftarg_t *target,
Nathan Scott204ab252006-01-11 20:50:22 +1100245 xfs_off_t range_base,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700246 size_t range_length,
Nathan Scottce8e9222006-01-11 15:39:08 +1100247 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700248{
249 /*
Nathan Scottce8e9222006-01-11 15:39:08 +1100250 * We don't want certain flags to appear in b_flags.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100252 flags &= ~(XBF_LOCK|XBF_MAPPED|XBF_DONT_BLOCK|XBF_READ_AHEAD);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700253
Nathan Scottce8e9222006-01-11 15:39:08 +1100254 memset(bp, 0, sizeof(xfs_buf_t));
255 atomic_set(&bp->b_hold, 1);
256 init_MUTEX_LOCKED(&bp->b_iodonesema);
257 INIT_LIST_HEAD(&bp->b_list);
258 INIT_LIST_HEAD(&bp->b_hash_list);
259 init_MUTEX_LOCKED(&bp->b_sema); /* held, no waiters */
260 XB_SET_OWNER(bp);
261 bp->b_target = target;
262 bp->b_file_offset = range_base;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263 /*
264 * Set buffer_length and count_desired to the same value initially.
265 * I/O routines should use count_desired, which will be the same in
266 * most cases but may be reset (e.g. XFS recovery).
267 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100268 bp->b_buffer_length = bp->b_count_desired = range_length;
269 bp->b_flags = flags;
270 bp->b_bn = XFS_BUF_DADDR_NULL;
271 atomic_set(&bp->b_pin_count, 0);
272 init_waitqueue_head(&bp->b_waiters);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700273
Nathan Scottce8e9222006-01-11 15:39:08 +1100274 XFS_STATS_INC(xb_create);
275 XB_TRACE(bp, "initialize", target);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276}
277
278/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100279 * Allocate a page array capable of holding a specified number
280 * of pages, and point the page buf at it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700281 */
282STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100283_xfs_buf_get_pages(
284 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700285 int page_count,
Nathan Scottce8e9222006-01-11 15:39:08 +1100286 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287{
288 /* Make sure that we have a page list */
Nathan Scottce8e9222006-01-11 15:39:08 +1100289 if (bp->b_pages == NULL) {
290 bp->b_offset = xfs_buf_poff(bp->b_file_offset);
291 bp->b_page_count = page_count;
292 if (page_count <= XB_PAGES) {
293 bp->b_pages = bp->b_page_array;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700294 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100295 bp->b_pages = kmem_alloc(sizeof(struct page *) *
296 page_count, xb_to_km(flags));
297 if (bp->b_pages == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700298 return -ENOMEM;
299 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100300 memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301 }
302 return 0;
303}
304
305/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100306 * Frees b_pages if it was allocated.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700307 */
308STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +1100309_xfs_buf_free_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310 xfs_buf_t *bp)
311{
Nathan Scottce8e9222006-01-11 15:39:08 +1100312 if (bp->b_pages != bp->b_page_array) {
313 kmem_free(bp->b_pages,
314 bp->b_page_count * sizeof(struct page *));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700315 }
316}
317
318/*
319 * Releases the specified buffer.
320 *
321 * The modification state of any associated pages is left unchanged.
Nathan Scottce8e9222006-01-11 15:39:08 +1100322 * The buffer most not be on any hash - use xfs_buf_rele instead for
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323 * hashed and refcounted buffers
324 */
325void
Nathan Scottce8e9222006-01-11 15:39:08 +1100326xfs_buf_free(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700327 xfs_buf_t *bp)
328{
Nathan Scottce8e9222006-01-11 15:39:08 +1100329 XB_TRACE(bp, "free", 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700330
Nathan Scottce8e9222006-01-11 15:39:08 +1100331 ASSERT(list_empty(&bp->b_hash_list));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000333 if (bp->b_flags & (_XBF_PAGE_CACHE|_XBF_PAGES)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700334 uint i;
335
Nathan Scottce8e9222006-01-11 15:39:08 +1100336 if ((bp->b_flags & XBF_MAPPED) && (bp->b_page_count > 1))
337 free_address(bp->b_addr - bp->b_offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700338
Nathan Scott948ecdb2006-09-28 11:03:13 +1000339 for (i = 0; i < bp->b_page_count; i++) {
340 struct page *page = bp->b_pages[i];
341
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000342 if (bp->b_flags & _XBF_PAGE_CACHE)
343 ASSERT(!PagePrivate(page));
Nathan Scott948ecdb2006-09-28 11:03:13 +1000344 page_cache_release(page);
345 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100346 _xfs_buf_free_pages(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347 }
348
Nathan Scottce8e9222006-01-11 15:39:08 +1100349 xfs_buf_deallocate(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350}
351
352/*
353 * Finds all pages for buffer in question and builds it's page list.
354 */
355STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100356_xfs_buf_lookup_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357 xfs_buf_t *bp,
358 uint flags)
359{
Nathan Scottce8e9222006-01-11 15:39:08 +1100360 struct address_space *mapping = bp->b_target->bt_mapping;
361 size_t blocksize = bp->b_target->bt_bsize;
362 size_t size = bp->b_count_desired;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363 size_t nbytes, offset;
Nathan Scottce8e9222006-01-11 15:39:08 +1100364 gfp_t gfp_mask = xb_to_gfp(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700365 unsigned short page_count, i;
366 pgoff_t first;
Nathan Scott204ab252006-01-11 20:50:22 +1100367 xfs_off_t end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700368 int error;
369
Nathan Scottce8e9222006-01-11 15:39:08 +1100370 end = bp->b_file_offset + bp->b_buffer_length;
371 page_count = xfs_buf_btoc(end) - xfs_buf_btoct(bp->b_file_offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700372
Nathan Scottce8e9222006-01-11 15:39:08 +1100373 error = _xfs_buf_get_pages(bp, page_count, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374 if (unlikely(error))
375 return error;
Nathan Scottce8e9222006-01-11 15:39:08 +1100376 bp->b_flags |= _XBF_PAGE_CACHE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377
Nathan Scottce8e9222006-01-11 15:39:08 +1100378 offset = bp->b_offset;
379 first = bp->b_file_offset >> PAGE_CACHE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700380
Nathan Scottce8e9222006-01-11 15:39:08 +1100381 for (i = 0; i < bp->b_page_count; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382 struct page *page;
383 uint retries = 0;
384
385 retry:
386 page = find_or_create_page(mapping, first + i, gfp_mask);
387 if (unlikely(page == NULL)) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100388 if (flags & XBF_READ_AHEAD) {
389 bp->b_page_count = i;
390 for (i = 0; i < bp->b_page_count; i++)
391 unlock_page(bp->b_pages[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700392 return -ENOMEM;
393 }
394
395 /*
396 * This could deadlock.
397 *
398 * But until all the XFS lowlevel code is revamped to
399 * handle buffer allocation failures we can't do much.
400 */
401 if (!(++retries % 100))
402 printk(KERN_ERR
403 "XFS: possible memory allocation "
404 "deadlock in %s (mode:0x%x)\n",
405 __FUNCTION__, gfp_mask);
406
Nathan Scottce8e9222006-01-11 15:39:08 +1100407 XFS_STATS_INC(xb_page_retries);
Christoph Hellwig23ea4032005-06-21 15:14:01 +1000408 xfsbufd_wakeup(0, gfp_mask);
Andrew Morton3fcfab12006-10-19 23:28:16 -0700409 congestion_wait(WRITE, HZ/50);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700410 goto retry;
411 }
412
Nathan Scottce8e9222006-01-11 15:39:08 +1100413 XFS_STATS_INC(xb_page_found);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700414
415 nbytes = min_t(size_t, size, PAGE_CACHE_SIZE - offset);
416 size -= nbytes;
417
Nathan Scott948ecdb2006-09-28 11:03:13 +1000418 ASSERT(!PagePrivate(page));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419 if (!PageUptodate(page)) {
420 page_count--;
421 if (blocksize >= PAGE_CACHE_SIZE) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100422 if (flags & XBF_READ)
423 bp->b_locked = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424 } else if (!PagePrivate(page)) {
425 if (test_page_region(page, offset, nbytes))
426 page_count++;
427 }
428 }
429
Nathan Scottce8e9222006-01-11 15:39:08 +1100430 bp->b_pages[i] = page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700431 offset = 0;
432 }
433
Nathan Scottce8e9222006-01-11 15:39:08 +1100434 if (!bp->b_locked) {
435 for (i = 0; i < bp->b_page_count; i++)
436 unlock_page(bp->b_pages[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437 }
438
Nathan Scottce8e9222006-01-11 15:39:08 +1100439 if (page_count == bp->b_page_count)
440 bp->b_flags |= XBF_DONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700441
Nathan Scottce8e9222006-01-11 15:39:08 +1100442 XB_TRACE(bp, "lookup_pages", (long)page_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443 return error;
444}
445
446/*
447 * Map buffer into kernel address-space if nessecary.
448 */
449STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100450_xfs_buf_map_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700451 xfs_buf_t *bp,
452 uint flags)
453{
454 /* A single page buffer is always mappable */
Nathan Scottce8e9222006-01-11 15:39:08 +1100455 if (bp->b_page_count == 1) {
456 bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
457 bp->b_flags |= XBF_MAPPED;
458 } else if (flags & XBF_MAPPED) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700459 if (as_list_len > 64)
460 purge_addresses();
Nathan Scottce8e9222006-01-11 15:39:08 +1100461 bp->b_addr = vmap(bp->b_pages, bp->b_page_count,
462 VM_MAP, PAGE_KERNEL);
463 if (unlikely(bp->b_addr == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464 return -ENOMEM;
Nathan Scottce8e9222006-01-11 15:39:08 +1100465 bp->b_addr += bp->b_offset;
466 bp->b_flags |= XBF_MAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467 }
468
469 return 0;
470}
471
472/*
473 * Finding and Reading Buffers
474 */
475
476/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100477 * Look up, and creates if absent, a lockable buffer for
Linus Torvalds1da177e2005-04-16 15:20:36 -0700478 * a given range of an inode. The buffer is returned
479 * locked. If other overlapping buffers exist, they are
480 * released before the new buffer is created and locked,
481 * which may imply that this call will block until those buffers
482 * are unlocked. No I/O is implied by this call.
483 */
484xfs_buf_t *
Nathan Scottce8e9222006-01-11 15:39:08 +1100485_xfs_buf_find(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486 xfs_buftarg_t *btp, /* block device target */
Nathan Scott204ab252006-01-11 20:50:22 +1100487 xfs_off_t ioff, /* starting offset of range */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488 size_t isize, /* length of range */
Nathan Scottce8e9222006-01-11 15:39:08 +1100489 xfs_buf_flags_t flags,
490 xfs_buf_t *new_bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700491{
Nathan Scott204ab252006-01-11 20:50:22 +1100492 xfs_off_t range_base;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700493 size_t range_length;
494 xfs_bufhash_t *hash;
Nathan Scottce8e9222006-01-11 15:39:08 +1100495 xfs_buf_t *bp, *n;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700496
497 range_base = (ioff << BBSHIFT);
498 range_length = (isize << BBSHIFT);
499
500 /* Check for IOs smaller than the sector size / not sector aligned */
Nathan Scottce8e9222006-01-11 15:39:08 +1100501 ASSERT(!(range_length < (1 << btp->bt_sshift)));
Nathan Scott204ab252006-01-11 20:50:22 +1100502 ASSERT(!(range_base & (xfs_off_t)btp->bt_smask));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503
504 hash = &btp->bt_hash[hash_long((unsigned long)ioff, btp->bt_hashshift)];
505
506 spin_lock(&hash->bh_lock);
507
Nathan Scottce8e9222006-01-11 15:39:08 +1100508 list_for_each_entry_safe(bp, n, &hash->bh_list, b_hash_list) {
509 ASSERT(btp == bp->b_target);
510 if (bp->b_file_offset == range_base &&
511 bp->b_buffer_length == range_length) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700512 /*
Nathan Scottce8e9222006-01-11 15:39:08 +1100513 * If we look at something, bring it to the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700514 * front of the list for next time.
515 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100516 atomic_inc(&bp->b_hold);
517 list_move(&bp->b_hash_list, &hash->bh_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518 goto found;
519 }
520 }
521
522 /* No match found */
Nathan Scottce8e9222006-01-11 15:39:08 +1100523 if (new_bp) {
524 _xfs_buf_initialize(new_bp, btp, range_base,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525 range_length, flags);
Nathan Scottce8e9222006-01-11 15:39:08 +1100526 new_bp->b_hash = hash;
527 list_add(&new_bp->b_hash_list, &hash->bh_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100529 XFS_STATS_INC(xb_miss_locked);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530 }
531
532 spin_unlock(&hash->bh_lock);
Nathan Scottce8e9222006-01-11 15:39:08 +1100533 return new_bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700534
535found:
536 spin_unlock(&hash->bh_lock);
537
538 /* Attempt to get the semaphore without sleeping,
539 * if this does not work then we need to drop the
540 * spinlock and do a hard attempt on the semaphore.
541 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100542 if (down_trylock(&bp->b_sema)) {
543 if (!(flags & XBF_TRYLOCK)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544 /* wait for buffer ownership */
Nathan Scottce8e9222006-01-11 15:39:08 +1100545 XB_TRACE(bp, "get_lock", 0);
546 xfs_buf_lock(bp);
547 XFS_STATS_INC(xb_get_locked_waited);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700548 } else {
549 /* We asked for a trylock and failed, no need
550 * to look at file offset and length here, we
Nathan Scottce8e9222006-01-11 15:39:08 +1100551 * know that this buffer at least overlaps our
552 * buffer and is locked, therefore our buffer
553 * either does not exist, or is this buffer.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100555 xfs_buf_rele(bp);
556 XFS_STATS_INC(xb_busy_locked);
557 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700558 }
559 } else {
560 /* trylock worked */
Nathan Scottce8e9222006-01-11 15:39:08 +1100561 XB_SET_OWNER(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700562 }
563
Nathan Scottce8e9222006-01-11 15:39:08 +1100564 if (bp->b_flags & XBF_STALE) {
565 ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
566 bp->b_flags &= XBF_MAPPED;
David Chinner2f926582005-09-05 08:33:35 +1000567 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100568 XB_TRACE(bp, "got_lock", 0);
569 XFS_STATS_INC(xb_get_locked);
570 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700571}
572
573/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100574 * Assembles a buffer covering the specified range.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700575 * Storage in memory for all portions of the buffer will be allocated,
576 * although backing storage may not be.
577 */
578xfs_buf_t *
Nathan Scottce8e9222006-01-11 15:39:08 +1100579xfs_buf_get_flags(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580 xfs_buftarg_t *target,/* target for buffer */
Nathan Scott204ab252006-01-11 20:50:22 +1100581 xfs_off_t ioff, /* starting offset of range */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700582 size_t isize, /* length of range */
Nathan Scottce8e9222006-01-11 15:39:08 +1100583 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584{
Nathan Scottce8e9222006-01-11 15:39:08 +1100585 xfs_buf_t *bp, *new_bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586 int error = 0, i;
587
Nathan Scottce8e9222006-01-11 15:39:08 +1100588 new_bp = xfs_buf_allocate(flags);
589 if (unlikely(!new_bp))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590 return NULL;
591
Nathan Scottce8e9222006-01-11 15:39:08 +1100592 bp = _xfs_buf_find(target, ioff, isize, flags, new_bp);
593 if (bp == new_bp) {
594 error = _xfs_buf_lookup_pages(bp, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595 if (error)
596 goto no_buffer;
597 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100598 xfs_buf_deallocate(new_bp);
599 if (unlikely(bp == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600 return NULL;
601 }
602
Nathan Scottce8e9222006-01-11 15:39:08 +1100603 for (i = 0; i < bp->b_page_count; i++)
604 mark_page_accessed(bp->b_pages[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605
Nathan Scottce8e9222006-01-11 15:39:08 +1100606 if (!(bp->b_flags & XBF_MAPPED)) {
607 error = _xfs_buf_map_pages(bp, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700608 if (unlikely(error)) {
609 printk(KERN_WARNING "%s: failed to map pages\n",
610 __FUNCTION__);
611 goto no_buffer;
612 }
613 }
614
Nathan Scottce8e9222006-01-11 15:39:08 +1100615 XFS_STATS_INC(xb_get);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616
617 /*
618 * Always fill in the block number now, the mapped cases can do
619 * their own overlay of this later.
620 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100621 bp->b_bn = ioff;
622 bp->b_count_desired = bp->b_buffer_length;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623
Nathan Scottce8e9222006-01-11 15:39:08 +1100624 XB_TRACE(bp, "get", (unsigned long)flags);
625 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700626
627 no_buffer:
Nathan Scottce8e9222006-01-11 15:39:08 +1100628 if (flags & (XBF_LOCK | XBF_TRYLOCK))
629 xfs_buf_unlock(bp);
630 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631 return NULL;
632}
633
634xfs_buf_t *
635xfs_buf_read_flags(
636 xfs_buftarg_t *target,
Nathan Scott204ab252006-01-11 20:50:22 +1100637 xfs_off_t ioff,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638 size_t isize,
Nathan Scottce8e9222006-01-11 15:39:08 +1100639 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700640{
Nathan Scottce8e9222006-01-11 15:39:08 +1100641 xfs_buf_t *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700642
Nathan Scottce8e9222006-01-11 15:39:08 +1100643 flags |= XBF_READ;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644
Nathan Scottce8e9222006-01-11 15:39:08 +1100645 bp = xfs_buf_get_flags(target, ioff, isize, flags);
646 if (bp) {
647 if (!XFS_BUF_ISDONE(bp)) {
648 XB_TRACE(bp, "read", (unsigned long)flags);
649 XFS_STATS_INC(xb_get_read);
650 xfs_buf_iostart(bp, flags);
651 } else if (flags & XBF_ASYNC) {
652 XB_TRACE(bp, "read_async", (unsigned long)flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700653 /*
654 * Read ahead call which is already satisfied,
655 * drop the buffer
656 */
657 goto no_buffer;
658 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100659 XB_TRACE(bp, "read_done", (unsigned long)flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700660 /* We do not want read in the flags */
Nathan Scottce8e9222006-01-11 15:39:08 +1100661 bp->b_flags &= ~XBF_READ;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662 }
663 }
664
Nathan Scottce8e9222006-01-11 15:39:08 +1100665 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666
667 no_buffer:
Nathan Scottce8e9222006-01-11 15:39:08 +1100668 if (flags & (XBF_LOCK | XBF_TRYLOCK))
669 xfs_buf_unlock(bp);
670 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671 return NULL;
672}
673
674/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100675 * If we are not low on memory then do the readahead in a deadlock
676 * safe manner.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677 */
678void
Nathan Scottce8e9222006-01-11 15:39:08 +1100679xfs_buf_readahead(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700680 xfs_buftarg_t *target,
Nathan Scott204ab252006-01-11 20:50:22 +1100681 xfs_off_t ioff,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700682 size_t isize,
Nathan Scottce8e9222006-01-11 15:39:08 +1100683 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700684{
685 struct backing_dev_info *bdi;
686
Nathan Scottce8e9222006-01-11 15:39:08 +1100687 bdi = target->bt_mapping->backing_dev_info;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700688 if (bdi_read_congested(bdi))
689 return;
690
Nathan Scottce8e9222006-01-11 15:39:08 +1100691 flags |= (XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700692 xfs_buf_read_flags(target, ioff, isize, flags);
693}
694
695xfs_buf_t *
Nathan Scottce8e9222006-01-11 15:39:08 +1100696xfs_buf_get_empty(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700697 size_t len,
698 xfs_buftarg_t *target)
699{
Nathan Scottce8e9222006-01-11 15:39:08 +1100700 xfs_buf_t *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700701
Nathan Scottce8e9222006-01-11 15:39:08 +1100702 bp = xfs_buf_allocate(0);
703 if (bp)
704 _xfs_buf_initialize(bp, target, 0, len, 0);
705 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706}
707
708static inline struct page *
709mem_to_page(
710 void *addr)
711{
712 if (((unsigned long)addr < VMALLOC_START) ||
713 ((unsigned long)addr >= VMALLOC_END)) {
714 return virt_to_page(addr);
715 } else {
716 return vmalloc_to_page(addr);
717 }
718}
719
720int
Nathan Scottce8e9222006-01-11 15:39:08 +1100721xfs_buf_associate_memory(
722 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700723 void *mem,
724 size_t len)
725{
726 int rval;
727 int i = 0;
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100728 unsigned long pageaddr;
729 unsigned long offset;
730 size_t buflen;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700731 int page_count;
732
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100733 pageaddr = (unsigned long)mem & PAGE_CACHE_MASK;
734 offset = (unsigned long)mem - pageaddr;
735 buflen = PAGE_CACHE_ALIGN(len + offset);
736 page_count = buflen >> PAGE_CACHE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700737
738 /* Free any previous set of page pointers */
Nathan Scottce8e9222006-01-11 15:39:08 +1100739 if (bp->b_pages)
740 _xfs_buf_free_pages(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700741
Nathan Scottce8e9222006-01-11 15:39:08 +1100742 bp->b_pages = NULL;
743 bp->b_addr = mem;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700744
Nathan Scottce8e9222006-01-11 15:39:08 +1100745 rval = _xfs_buf_get_pages(bp, page_count, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746 if (rval)
747 return rval;
748
Nathan Scottce8e9222006-01-11 15:39:08 +1100749 bp->b_offset = offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700750
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100751 for (i = 0; i < bp->b_page_count; i++) {
752 bp->b_pages[i] = mem_to_page((void *)pageaddr);
753 pageaddr += PAGE_CACHE_SIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700754 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100755 bp->b_locked = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100757 bp->b_count_desired = len;
758 bp->b_buffer_length = buflen;
Nathan Scottce8e9222006-01-11 15:39:08 +1100759 bp->b_flags |= XBF_MAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700760
761 return 0;
762}
763
764xfs_buf_t *
Nathan Scottce8e9222006-01-11 15:39:08 +1100765xfs_buf_get_noaddr(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700766 size_t len,
767 xfs_buftarg_t *target)
768{
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000769 unsigned long page_count = PAGE_ALIGN(len) >> PAGE_SHIFT;
770 int error, i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700771 xfs_buf_t *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700772
Nathan Scottce8e9222006-01-11 15:39:08 +1100773 bp = xfs_buf_allocate(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774 if (unlikely(bp == NULL))
775 goto fail;
Nathan Scottce8e9222006-01-11 15:39:08 +1100776 _xfs_buf_initialize(bp, target, 0, len, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700777
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000778 error = _xfs_buf_get_pages(bp, page_count, 0);
779 if (error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700780 goto fail_free_buf;
781
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000782 for (i = 0; i < page_count; i++) {
783 bp->b_pages[i] = alloc_page(GFP_KERNEL);
784 if (!bp->b_pages[i])
785 goto fail_free_mem;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700786 }
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000787 bp->b_flags |= _XBF_PAGES;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700788
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000789 error = _xfs_buf_map_pages(bp, XBF_MAPPED);
790 if (unlikely(error)) {
791 printk(KERN_WARNING "%s: failed to map pages\n",
792 __FUNCTION__);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700793 goto fail_free_mem;
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000794 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700795
Nathan Scottce8e9222006-01-11 15:39:08 +1100796 xfs_buf_unlock(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000798 XB_TRACE(bp, "no_daddr", len);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799 return bp;
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000800
Linus Torvalds1da177e2005-04-16 15:20:36 -0700801 fail_free_mem:
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000802 while (--i >= 0)
803 __free_page(bp->b_pages[i]);
Christoph Hellwigca165b82007-05-24 15:21:11 +1000804 _xfs_buf_free_pages(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700805 fail_free_buf:
Christoph Hellwigca165b82007-05-24 15:21:11 +1000806 xfs_buf_deallocate(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700807 fail:
808 return NULL;
809}
810
811/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812 * Increment reference count on buffer, to hold the buffer concurrently
813 * with another thread which may release (free) the buffer asynchronously.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700814 * Must hold the buffer already to call this function.
815 */
816void
Nathan Scottce8e9222006-01-11 15:39:08 +1100817xfs_buf_hold(
818 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700819{
Nathan Scottce8e9222006-01-11 15:39:08 +1100820 atomic_inc(&bp->b_hold);
821 XB_TRACE(bp, "hold", 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700822}
823
824/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100825 * Releases a hold on the specified buffer. If the
826 * the hold count is 1, calls xfs_buf_free.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700827 */
828void
Nathan Scottce8e9222006-01-11 15:39:08 +1100829xfs_buf_rele(
830 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700831{
Nathan Scottce8e9222006-01-11 15:39:08 +1100832 xfs_bufhash_t *hash = bp->b_hash;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700833
Nathan Scottce8e9222006-01-11 15:39:08 +1100834 XB_TRACE(bp, "rele", bp->b_relse);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700835
Nathan Scottfad3aa12006-02-01 12:14:52 +1100836 if (unlikely(!hash)) {
837 ASSERT(!bp->b_relse);
838 if (atomic_dec_and_test(&bp->b_hold))
839 xfs_buf_free(bp);
840 return;
841 }
842
Nathan Scottce8e9222006-01-11 15:39:08 +1100843 if (atomic_dec_and_lock(&bp->b_hold, &hash->bh_lock)) {
844 if (bp->b_relse) {
845 atomic_inc(&bp->b_hold);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700846 spin_unlock(&hash->bh_lock);
Nathan Scottce8e9222006-01-11 15:39:08 +1100847 (*(bp->b_relse)) (bp);
848 } else if (bp->b_flags & XBF_FS_MANAGED) {
Christoph Hellwig7f14d0a2005-11-02 15:09:35 +1100849 spin_unlock(&hash->bh_lock);
850 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100851 ASSERT(!(bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)));
852 list_del_init(&bp->b_hash_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700853 spin_unlock(&hash->bh_lock);
Nathan Scottce8e9222006-01-11 15:39:08 +1100854 xfs_buf_free(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700855 }
David Chinner2f926582005-09-05 08:33:35 +1000856 } else {
857 /*
858 * Catch reference count leaks
859 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100860 ASSERT(atomic_read(&bp->b_hold) >= 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700861 }
862}
863
864
865/*
866 * Mutual exclusion on buffers. Locking model:
867 *
868 * Buffers associated with inodes for which buffer locking
869 * is not enabled are not protected by semaphores, and are
870 * assumed to be exclusively owned by the caller. There is a
871 * spinlock in the buffer, used by the caller when concurrent
872 * access is possible.
873 */
874
875/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100876 * Locks a buffer object, if it is not already locked.
877 * Note that this in no way locks the underlying pages, so it is only
878 * useful for synchronizing concurrent use of buffer objects, not for
879 * synchronizing independent access to the underlying pages.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700880 */
881int
Nathan Scottce8e9222006-01-11 15:39:08 +1100882xfs_buf_cond_lock(
883 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700884{
885 int locked;
886
Nathan Scottce8e9222006-01-11 15:39:08 +1100887 locked = down_trylock(&bp->b_sema) == 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700888 if (locked) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100889 XB_SET_OWNER(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700890 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100891 XB_TRACE(bp, "cond_lock", (long)locked);
892 return locked ? 0 : -EBUSY;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700893}
894
895#if defined(DEBUG) || defined(XFS_BLI_TRACE)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700896int
Nathan Scottce8e9222006-01-11 15:39:08 +1100897xfs_buf_lock_value(
898 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700899{
Nathan Scottce8e9222006-01-11 15:39:08 +1100900 return atomic_read(&bp->b_sema.count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901}
902#endif
903
904/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100905 * Locks a buffer object.
906 * Note that this in no way locks the underlying pages, so it is only
907 * useful for synchronizing concurrent use of buffer objects, not for
908 * synchronizing independent access to the underlying pages.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700909 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100910void
911xfs_buf_lock(
912 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700913{
Nathan Scottce8e9222006-01-11 15:39:08 +1100914 XB_TRACE(bp, "lock", 0);
915 if (atomic_read(&bp->b_io_remaining))
916 blk_run_address_space(bp->b_target->bt_mapping);
917 down(&bp->b_sema);
918 XB_SET_OWNER(bp);
919 XB_TRACE(bp, "locked", 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700920}
921
922/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100923 * Releases the lock on the buffer object.
David Chinner2f926582005-09-05 08:33:35 +1000924 * If the buffer is marked delwri but is not queued, do so before we
Nathan Scottce8e9222006-01-11 15:39:08 +1100925 * unlock the buffer as we need to set flags correctly. We also need to
David Chinner2f926582005-09-05 08:33:35 +1000926 * take a reference for the delwri queue because the unlocker is going to
927 * drop their's and they don't know we just queued it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700928 */
929void
Nathan Scottce8e9222006-01-11 15:39:08 +1100930xfs_buf_unlock(
931 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700932{
Nathan Scottce8e9222006-01-11 15:39:08 +1100933 if ((bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)) == XBF_DELWRI) {
934 atomic_inc(&bp->b_hold);
935 bp->b_flags |= XBF_ASYNC;
936 xfs_buf_delwri_queue(bp, 0);
David Chinner2f926582005-09-05 08:33:35 +1000937 }
938
Nathan Scottce8e9222006-01-11 15:39:08 +1100939 XB_CLEAR_OWNER(bp);
940 up(&bp->b_sema);
941 XB_TRACE(bp, "unlock", 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700942}
943
944
945/*
946 * Pinning Buffer Storage in Memory
Nathan Scottce8e9222006-01-11 15:39:08 +1100947 * Ensure that no attempt to force a buffer to disk will succeed.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700948 */
949void
Nathan Scottce8e9222006-01-11 15:39:08 +1100950xfs_buf_pin(
951 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700952{
Nathan Scottce8e9222006-01-11 15:39:08 +1100953 atomic_inc(&bp->b_pin_count);
954 XB_TRACE(bp, "pin", (long)bp->b_pin_count.counter);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700955}
956
Linus Torvalds1da177e2005-04-16 15:20:36 -0700957void
Nathan Scottce8e9222006-01-11 15:39:08 +1100958xfs_buf_unpin(
959 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700960{
Nathan Scottce8e9222006-01-11 15:39:08 +1100961 if (atomic_dec_and_test(&bp->b_pin_count))
962 wake_up_all(&bp->b_waiters);
963 XB_TRACE(bp, "unpin", (long)bp->b_pin_count.counter);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700964}
965
966int
Nathan Scottce8e9222006-01-11 15:39:08 +1100967xfs_buf_ispin(
968 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969{
Nathan Scottce8e9222006-01-11 15:39:08 +1100970 return atomic_read(&bp->b_pin_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700971}
972
Nathan Scottce8e9222006-01-11 15:39:08 +1100973STATIC void
974xfs_buf_wait_unpin(
975 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700976{
977 DECLARE_WAITQUEUE (wait, current);
978
Nathan Scottce8e9222006-01-11 15:39:08 +1100979 if (atomic_read(&bp->b_pin_count) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700980 return;
981
Nathan Scottce8e9222006-01-11 15:39:08 +1100982 add_wait_queue(&bp->b_waiters, &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700983 for (;;) {
984 set_current_state(TASK_UNINTERRUPTIBLE);
Nathan Scottce8e9222006-01-11 15:39:08 +1100985 if (atomic_read(&bp->b_pin_count) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700986 break;
Nathan Scottce8e9222006-01-11 15:39:08 +1100987 if (atomic_read(&bp->b_io_remaining))
988 blk_run_address_space(bp->b_target->bt_mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700989 schedule();
990 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100991 remove_wait_queue(&bp->b_waiters, &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700992 set_current_state(TASK_RUNNING);
993}
994
995/*
996 * Buffer Utility Routines
997 */
998
Linus Torvalds1da177e2005-04-16 15:20:36 -0700999STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001000xfs_buf_iodone_work(
David Howellsc4028952006-11-22 14:57:56 +00001001 struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001002{
David Howellsc4028952006-11-22 14:57:56 +00001003 xfs_buf_t *bp =
1004 container_of(work, xfs_buf_t, b_iodone_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001005
David Chinner0bfefc42007-05-14 18:24:23 +10001006 /*
1007 * We can get an EOPNOTSUPP to ordered writes. Here we clear the
1008 * ordered flag and reissue them. Because we can't tell the higher
1009 * layers directly that they should not issue ordered I/O anymore, they
1010 * need to check if the ordered flag was cleared during I/O completion.
1011 */
1012 if ((bp->b_error == EOPNOTSUPP) &&
1013 (bp->b_flags & (XBF_ORDERED|XBF_ASYNC)) == (XBF_ORDERED|XBF_ASYNC)) {
1014 XB_TRACE(bp, "ordered_retry", bp->b_iodone);
1015 bp->b_flags &= ~XBF_ORDERED;
1016 xfs_buf_iorequest(bp);
1017 } else if (bp->b_iodone)
Nathan Scottce8e9222006-01-11 15:39:08 +11001018 (*(bp->b_iodone))(bp);
1019 else if (bp->b_flags & XBF_ASYNC)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001020 xfs_buf_relse(bp);
1021}
1022
1023void
Nathan Scottce8e9222006-01-11 15:39:08 +11001024xfs_buf_ioend(
1025 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001026 int schedule)
1027{
Lachlan McIlroy77be55a2007-11-23 16:31:00 +11001028 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
Nathan Scottce8e9222006-01-11 15:39:08 +11001029 if (bp->b_error == 0)
1030 bp->b_flags |= XBF_DONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031
Nathan Scottce8e9222006-01-11 15:39:08 +11001032 XB_TRACE(bp, "iodone", bp->b_iodone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001033
Nathan Scottce8e9222006-01-11 15:39:08 +11001034 if ((bp->b_iodone) || (bp->b_flags & XBF_ASYNC)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001035 if (schedule) {
David Howellsc4028952006-11-22 14:57:56 +00001036 INIT_WORK(&bp->b_iodone_work, xfs_buf_iodone_work);
Nathan Scottce8e9222006-01-11 15:39:08 +11001037 queue_work(xfslogd_workqueue, &bp->b_iodone_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001038 } else {
David Howellsc4028952006-11-22 14:57:56 +00001039 xfs_buf_iodone_work(&bp->b_iodone_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001040 }
1041 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +11001042 up(&bp->b_iodonesema);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001043 }
1044}
1045
Linus Torvalds1da177e2005-04-16 15:20:36 -07001046void
Nathan Scottce8e9222006-01-11 15:39:08 +11001047xfs_buf_ioerror(
1048 xfs_buf_t *bp,
1049 int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001050{
1051 ASSERT(error >= 0 && error <= 0xffff);
Nathan Scottce8e9222006-01-11 15:39:08 +11001052 bp->b_error = (unsigned short)error;
1053 XB_TRACE(bp, "ioerror", (unsigned long)error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001054}
1055
1056/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001057 * Initiate I/O on a buffer, based on the flags supplied.
1058 * The b_iodone routine in the buffer supplied will only be called
Linus Torvalds1da177e2005-04-16 15:20:36 -07001059 * when all of the subsidiary I/O requests, if any, have been completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001060 */
1061int
Nathan Scottce8e9222006-01-11 15:39:08 +11001062xfs_buf_iostart(
1063 xfs_buf_t *bp,
1064 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001065{
1066 int status = 0;
1067
Nathan Scottce8e9222006-01-11 15:39:08 +11001068 XB_TRACE(bp, "iostart", (unsigned long)flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001069
Nathan Scottce8e9222006-01-11 15:39:08 +11001070 if (flags & XBF_DELWRI) {
1071 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_ASYNC);
1072 bp->b_flags |= flags & (XBF_DELWRI | XBF_ASYNC);
1073 xfs_buf_delwri_queue(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001074 return status;
1075 }
1076
Nathan Scottce8e9222006-01-11 15:39:08 +11001077 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_ASYNC | XBF_DELWRI | \
1078 XBF_READ_AHEAD | _XBF_RUN_QUEUES);
1079 bp->b_flags |= flags & (XBF_READ | XBF_WRITE | XBF_ASYNC | \
1080 XBF_READ_AHEAD | _XBF_RUN_QUEUES);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001081
Nathan Scottce8e9222006-01-11 15:39:08 +11001082 BUG_ON(bp->b_bn == XFS_BUF_DADDR_NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001083
1084 /* For writes allow an alternate strategy routine to precede
1085 * the actual I/O request (which may not be issued at all in
1086 * a shutdown situation, for example).
1087 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001088 status = (flags & XBF_WRITE) ?
1089 xfs_buf_iostrategy(bp) : xfs_buf_iorequest(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001090
1091 /* Wait for I/O if we are not an async request.
1092 * Note: async I/O request completion will release the buffer,
1093 * and that can already be done by this point. So using the
1094 * buffer pointer from here on, after async I/O, is invalid.
1095 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001096 if (!status && !(flags & XBF_ASYNC))
1097 status = xfs_buf_iowait(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001098
1099 return status;
1100}
1101
David Chinner7989cb82007-02-10 18:34:56 +11001102STATIC_INLINE int
Nathan Scottce8e9222006-01-11 15:39:08 +11001103_xfs_buf_iolocked(
1104 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001105{
Nathan Scottce8e9222006-01-11 15:39:08 +11001106 ASSERT(bp->b_flags & (XBF_READ | XBF_WRITE));
1107 if (bp->b_flags & XBF_READ)
1108 return bp->b_locked;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001109 return 0;
1110}
1111
David Chinner7989cb82007-02-10 18:34:56 +11001112STATIC_INLINE void
Nathan Scottce8e9222006-01-11 15:39:08 +11001113_xfs_buf_ioend(
1114 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001115 int schedule)
1116{
Nathan Scottce8e9222006-01-11 15:39:08 +11001117 if (atomic_dec_and_test(&bp->b_io_remaining) == 1) {
1118 bp->b_locked = 0;
1119 xfs_buf_ioend(bp, schedule);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001120 }
1121}
1122
Al Viro782e3b32007-10-12 07:17:47 +01001123STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001124xfs_buf_bio_end_io(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001125 struct bio *bio,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001126 int error)
1127{
Nathan Scottce8e9222006-01-11 15:39:08 +11001128 xfs_buf_t *bp = (xfs_buf_t *)bio->bi_private;
1129 unsigned int blocksize = bp->b_target->bt_bsize;
Nathan Scotteedb5532005-09-02 16:39:56 +10001130 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001131
Linus Torvalds1da177e2005-04-16 15:20:36 -07001132 if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
Nathan Scottce8e9222006-01-11 15:39:08 +11001133 bp->b_error = EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001134
Nathan Scotteedb5532005-09-02 16:39:56 +10001135 do {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001136 struct page *page = bvec->bv_page;
1137
Nathan Scott948ecdb2006-09-28 11:03:13 +10001138 ASSERT(!PagePrivate(page));
Nathan Scottce8e9222006-01-11 15:39:08 +11001139 if (unlikely(bp->b_error)) {
1140 if (bp->b_flags & XBF_READ)
Nathan Scotteedb5532005-09-02 16:39:56 +10001141 ClearPageUptodate(page);
Nathan Scottce8e9222006-01-11 15:39:08 +11001142 } else if (blocksize >= PAGE_CACHE_SIZE) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001143 SetPageUptodate(page);
1144 } else if (!PagePrivate(page) &&
Nathan Scottce8e9222006-01-11 15:39:08 +11001145 (bp->b_flags & _XBF_PAGE_CACHE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001146 set_page_region(page, bvec->bv_offset, bvec->bv_len);
1147 }
1148
Nathan Scotteedb5532005-09-02 16:39:56 +10001149 if (--bvec >= bio->bi_io_vec)
1150 prefetchw(&bvec->bv_page->flags);
1151
Nathan Scottce8e9222006-01-11 15:39:08 +11001152 if (_xfs_buf_iolocked(bp)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001153 unlock_page(page);
1154 }
Nathan Scotteedb5532005-09-02 16:39:56 +10001155 } while (bvec >= bio->bi_io_vec);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001156
Nathan Scottce8e9222006-01-11 15:39:08 +11001157 _xfs_buf_ioend(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001158 bio_put(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001159}
1160
1161STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001162_xfs_buf_ioapply(
1163 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001164{
1165 int i, rw, map_i, total_nr_pages, nr_pages;
1166 struct bio *bio;
Nathan Scottce8e9222006-01-11 15:39:08 +11001167 int offset = bp->b_offset;
1168 int size = bp->b_count_desired;
1169 sector_t sector = bp->b_bn;
1170 unsigned int blocksize = bp->b_target->bt_bsize;
1171 int locking = _xfs_buf_iolocked(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001172
Nathan Scottce8e9222006-01-11 15:39:08 +11001173 total_nr_pages = bp->b_page_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001174 map_i = 0;
1175
Nathan Scottce8e9222006-01-11 15:39:08 +11001176 if (bp->b_flags & XBF_ORDERED) {
1177 ASSERT(!(bp->b_flags & XBF_READ));
Christoph Hellwigf538d4d2005-11-02 10:26:59 +11001178 rw = WRITE_BARRIER;
Nathan Scott51bdd702006-09-28 11:01:57 +10001179 } else if (bp->b_flags & _XBF_RUN_QUEUES) {
1180 ASSERT(!(bp->b_flags & XBF_READ_AHEAD));
1181 bp->b_flags &= ~_XBF_RUN_QUEUES;
1182 rw = (bp->b_flags & XBF_WRITE) ? WRITE_SYNC : READ_SYNC;
1183 } else {
1184 rw = (bp->b_flags & XBF_WRITE) ? WRITE :
1185 (bp->b_flags & XBF_READ_AHEAD) ? READA : READ;
Christoph Hellwigf538d4d2005-11-02 10:26:59 +11001186 }
1187
Nathan Scottce8e9222006-01-11 15:39:08 +11001188 /* Special code path for reading a sub page size buffer in --
Linus Torvalds1da177e2005-04-16 15:20:36 -07001189 * we populate up the whole page, and hence the other metadata
1190 * in the same page. This optimization is only valid when the
Nathan Scottce8e9222006-01-11 15:39:08 +11001191 * filesystem block size is not smaller than the page size.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001192 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001193 if ((bp->b_buffer_length < PAGE_CACHE_SIZE) &&
1194 (bp->b_flags & XBF_READ) && locking &&
1195 (blocksize >= PAGE_CACHE_SIZE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001196 bio = bio_alloc(GFP_NOIO, 1);
1197
Nathan Scottce8e9222006-01-11 15:39:08 +11001198 bio->bi_bdev = bp->b_target->bt_bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001199 bio->bi_sector = sector - (offset >> BBSHIFT);
Nathan Scottce8e9222006-01-11 15:39:08 +11001200 bio->bi_end_io = xfs_buf_bio_end_io;
1201 bio->bi_private = bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001202
Nathan Scottce8e9222006-01-11 15:39:08 +11001203 bio_add_page(bio, bp->b_pages[0], PAGE_CACHE_SIZE, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001204 size = 0;
1205
Nathan Scottce8e9222006-01-11 15:39:08 +11001206 atomic_inc(&bp->b_io_remaining);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001207
1208 goto submit_io;
1209 }
1210
1211 /* Lock down the pages which we need to for the request */
Nathan Scottce8e9222006-01-11 15:39:08 +11001212 if (locking && (bp->b_flags & XBF_WRITE) && (bp->b_locked == 0)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001213 for (i = 0; size; i++) {
1214 int nbytes = PAGE_CACHE_SIZE - offset;
Nathan Scottce8e9222006-01-11 15:39:08 +11001215 struct page *page = bp->b_pages[i];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001216
1217 if (nbytes > size)
1218 nbytes = size;
1219
1220 lock_page(page);
1221
1222 size -= nbytes;
1223 offset = 0;
1224 }
Nathan Scottce8e9222006-01-11 15:39:08 +11001225 offset = bp->b_offset;
1226 size = bp->b_count_desired;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001227 }
1228
1229next_chunk:
Nathan Scottce8e9222006-01-11 15:39:08 +11001230 atomic_inc(&bp->b_io_remaining);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001231 nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1232 if (nr_pages > total_nr_pages)
1233 nr_pages = total_nr_pages;
1234
1235 bio = bio_alloc(GFP_NOIO, nr_pages);
Nathan Scottce8e9222006-01-11 15:39:08 +11001236 bio->bi_bdev = bp->b_target->bt_bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001237 bio->bi_sector = sector;
Nathan Scottce8e9222006-01-11 15:39:08 +11001238 bio->bi_end_io = xfs_buf_bio_end_io;
1239 bio->bi_private = bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001240
1241 for (; size && nr_pages; nr_pages--, map_i++) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001242 int rbytes, nbytes = PAGE_CACHE_SIZE - offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001243
1244 if (nbytes > size)
1245 nbytes = size;
1246
Nathan Scottce8e9222006-01-11 15:39:08 +11001247 rbytes = bio_add_page(bio, bp->b_pages[map_i], nbytes, offset);
1248 if (rbytes < nbytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001249 break;
1250
1251 offset = 0;
1252 sector += nbytes >> BBSHIFT;
1253 size -= nbytes;
1254 total_nr_pages--;
1255 }
1256
1257submit_io:
1258 if (likely(bio->bi_size)) {
1259 submit_bio(rw, bio);
1260 if (size)
1261 goto next_chunk;
1262 } else {
1263 bio_put(bio);
Nathan Scottce8e9222006-01-11 15:39:08 +11001264 xfs_buf_ioerror(bp, EIO);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001265 }
1266}
1267
Linus Torvalds1da177e2005-04-16 15:20:36 -07001268int
Nathan Scottce8e9222006-01-11 15:39:08 +11001269xfs_buf_iorequest(
1270 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001271{
Nathan Scottce8e9222006-01-11 15:39:08 +11001272 XB_TRACE(bp, "iorequest", 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001273
Nathan Scottce8e9222006-01-11 15:39:08 +11001274 if (bp->b_flags & XBF_DELWRI) {
1275 xfs_buf_delwri_queue(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001276 return 0;
1277 }
1278
Nathan Scottce8e9222006-01-11 15:39:08 +11001279 if (bp->b_flags & XBF_WRITE) {
1280 xfs_buf_wait_unpin(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001281 }
1282
Nathan Scottce8e9222006-01-11 15:39:08 +11001283 xfs_buf_hold(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001284
1285 /* Set the count to 1 initially, this will stop an I/O
1286 * completion callout which happens before we have started
Nathan Scottce8e9222006-01-11 15:39:08 +11001287 * all the I/O from calling xfs_buf_ioend too early.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001288 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001289 atomic_set(&bp->b_io_remaining, 1);
1290 _xfs_buf_ioapply(bp);
1291 _xfs_buf_ioend(bp, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001292
Nathan Scottce8e9222006-01-11 15:39:08 +11001293 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001294 return 0;
1295}
1296
1297/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001298 * Waits for I/O to complete on the buffer supplied.
1299 * It returns immediately if no I/O is pending.
1300 * It returns the I/O error code, if any, or 0 if there was no error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001301 */
1302int
Nathan Scottce8e9222006-01-11 15:39:08 +11001303xfs_buf_iowait(
1304 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001305{
Nathan Scottce8e9222006-01-11 15:39:08 +11001306 XB_TRACE(bp, "iowait", 0);
1307 if (atomic_read(&bp->b_io_remaining))
1308 blk_run_address_space(bp->b_target->bt_mapping);
1309 down(&bp->b_iodonesema);
1310 XB_TRACE(bp, "iowaited", (long)bp->b_error);
1311 return bp->b_error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001312}
1313
Nathan Scottce8e9222006-01-11 15:39:08 +11001314xfs_caddr_t
1315xfs_buf_offset(
1316 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001317 size_t offset)
1318{
1319 struct page *page;
1320
Nathan Scottce8e9222006-01-11 15:39:08 +11001321 if (bp->b_flags & XBF_MAPPED)
1322 return XFS_BUF_PTR(bp) + offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001323
Nathan Scottce8e9222006-01-11 15:39:08 +11001324 offset += bp->b_offset;
1325 page = bp->b_pages[offset >> PAGE_CACHE_SHIFT];
1326 return (xfs_caddr_t)page_address(page) + (offset & (PAGE_CACHE_SIZE-1));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001327}
1328
1329/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001330 * Move data into or out of a buffer.
1331 */
1332void
Nathan Scottce8e9222006-01-11 15:39:08 +11001333xfs_buf_iomove(
1334 xfs_buf_t *bp, /* buffer to process */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001335 size_t boff, /* starting buffer offset */
1336 size_t bsize, /* length to copy */
1337 caddr_t data, /* data address */
Nathan Scottce8e9222006-01-11 15:39:08 +11001338 xfs_buf_rw_t mode) /* read/write/zero flag */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001339{
1340 size_t bend, cpoff, csize;
1341 struct page *page;
1342
1343 bend = boff + bsize;
1344 while (boff < bend) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001345 page = bp->b_pages[xfs_buf_btoct(boff + bp->b_offset)];
1346 cpoff = xfs_buf_poff(boff + bp->b_offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001347 csize = min_t(size_t,
Nathan Scottce8e9222006-01-11 15:39:08 +11001348 PAGE_CACHE_SIZE-cpoff, bp->b_count_desired-boff);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001349
1350 ASSERT(((csize + cpoff) <= PAGE_CACHE_SIZE));
1351
1352 switch (mode) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001353 case XBRW_ZERO:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001354 memset(page_address(page) + cpoff, 0, csize);
1355 break;
Nathan Scottce8e9222006-01-11 15:39:08 +11001356 case XBRW_READ:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001357 memcpy(data, page_address(page) + cpoff, csize);
1358 break;
Nathan Scottce8e9222006-01-11 15:39:08 +11001359 case XBRW_WRITE:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001360 memcpy(page_address(page) + cpoff, data, csize);
1361 }
1362
1363 boff += csize;
1364 data += csize;
1365 }
1366}
1367
1368/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001369 * Handling of buffer targets (buftargs).
Linus Torvalds1da177e2005-04-16 15:20:36 -07001370 */
1371
1372/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001373 * Wait for any bufs with callbacks that have been submitted but
1374 * have not yet returned... walk the hash list for the target.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001375 */
1376void
1377xfs_wait_buftarg(
1378 xfs_buftarg_t *btp)
1379{
1380 xfs_buf_t *bp, *n;
1381 xfs_bufhash_t *hash;
1382 uint i;
1383
1384 for (i = 0; i < (1 << btp->bt_hashshift); i++) {
1385 hash = &btp->bt_hash[i];
1386again:
1387 spin_lock(&hash->bh_lock);
Nathan Scottce8e9222006-01-11 15:39:08 +11001388 list_for_each_entry_safe(bp, n, &hash->bh_list, b_hash_list) {
1389 ASSERT(btp == bp->b_target);
1390 if (!(bp->b_flags & XBF_FS_MANAGED)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001391 spin_unlock(&hash->bh_lock);
David Chinner2f926582005-09-05 08:33:35 +10001392 /*
1393 * Catch superblock reference count leaks
1394 * immediately
1395 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001396 BUG_ON(bp->b_bn == 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001397 delay(100);
1398 goto again;
1399 }
1400 }
1401 spin_unlock(&hash->bh_lock);
1402 }
1403}
1404
1405/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001406 * Allocate buffer hash table for a given target.
1407 * For devices containing metadata (i.e. not the log/realtime devices)
1408 * we need to allocate a much larger hash table.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001409 */
1410STATIC void
1411xfs_alloc_bufhash(
1412 xfs_buftarg_t *btp,
1413 int external)
1414{
1415 unsigned int i;
1416
1417 btp->bt_hashshift = external ? 3 : 8; /* 8 or 256 buckets */
1418 btp->bt_hashmask = (1 << btp->bt_hashshift) - 1;
1419 btp->bt_hash = kmem_zalloc((1 << btp->bt_hashshift) *
Vlad Apostolov93c189c2006-11-11 18:03:49 +11001420 sizeof(xfs_bufhash_t), KM_SLEEP | KM_LARGE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001421 for (i = 0; i < (1 << btp->bt_hashshift); i++) {
1422 spin_lock_init(&btp->bt_hash[i].bh_lock);
1423 INIT_LIST_HEAD(&btp->bt_hash[i].bh_list);
1424 }
1425}
1426
1427STATIC void
1428xfs_free_bufhash(
1429 xfs_buftarg_t *btp)
1430{
Nathan Scottce8e9222006-01-11 15:39:08 +11001431 kmem_free(btp->bt_hash, (1<<btp->bt_hashshift) * sizeof(xfs_bufhash_t));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001432 btp->bt_hash = NULL;
1433}
1434
David Chinnera6867a62006-01-11 15:37:58 +11001435/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001436 * buftarg list for delwrite queue processing
David Chinnera6867a62006-01-11 15:37:58 +11001437 */
Tim Shimmine6a0e9c2007-05-08 13:49:59 +10001438static LIST_HEAD(xfs_buftarg_list);
David Chinner7989cb82007-02-10 18:34:56 +11001439static DEFINE_SPINLOCK(xfs_buftarg_lock);
David Chinnera6867a62006-01-11 15:37:58 +11001440
1441STATIC void
1442xfs_register_buftarg(
1443 xfs_buftarg_t *btp)
1444{
1445 spin_lock(&xfs_buftarg_lock);
1446 list_add(&btp->bt_list, &xfs_buftarg_list);
1447 spin_unlock(&xfs_buftarg_lock);
1448}
1449
1450STATIC void
1451xfs_unregister_buftarg(
1452 xfs_buftarg_t *btp)
1453{
1454 spin_lock(&xfs_buftarg_lock);
1455 list_del(&btp->bt_list);
1456 spin_unlock(&xfs_buftarg_lock);
1457}
1458
Linus Torvalds1da177e2005-04-16 15:20:36 -07001459void
1460xfs_free_buftarg(
1461 xfs_buftarg_t *btp,
1462 int external)
1463{
1464 xfs_flush_buftarg(btp, 1);
David Chinnerf4a9f282007-06-05 16:24:27 +10001465 xfs_blkdev_issue_flush(btp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001466 if (external)
Nathan Scottce8e9222006-01-11 15:39:08 +11001467 xfs_blkdev_put(btp->bt_bdev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001468 xfs_free_bufhash(btp);
Nathan Scottce8e9222006-01-11 15:39:08 +11001469 iput(btp->bt_mapping->host);
David Chinnera6867a62006-01-11 15:37:58 +11001470
Nathan Scottce8e9222006-01-11 15:39:08 +11001471 /* Unregister the buftarg first so that we don't get a
1472 * wakeup finding a non-existent task
1473 */
David Chinnera6867a62006-01-11 15:37:58 +11001474 xfs_unregister_buftarg(btp);
1475 kthread_stop(btp->bt_task);
1476
Linus Torvalds1da177e2005-04-16 15:20:36 -07001477 kmem_free(btp, sizeof(*btp));
1478}
1479
Linus Torvalds1da177e2005-04-16 15:20:36 -07001480STATIC int
1481xfs_setsize_buftarg_flags(
1482 xfs_buftarg_t *btp,
1483 unsigned int blocksize,
1484 unsigned int sectorsize,
1485 int verbose)
1486{
Nathan Scottce8e9222006-01-11 15:39:08 +11001487 btp->bt_bsize = blocksize;
1488 btp->bt_sshift = ffs(sectorsize) - 1;
1489 btp->bt_smask = sectorsize - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001490
Nathan Scottce8e9222006-01-11 15:39:08 +11001491 if (set_blocksize(btp->bt_bdev, sectorsize)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001492 printk(KERN_WARNING
1493 "XFS: Cannot set_blocksize to %u on device %s\n",
1494 sectorsize, XFS_BUFTARG_NAME(btp));
1495 return EINVAL;
1496 }
1497
1498 if (verbose &&
1499 (PAGE_CACHE_SIZE / BITS_PER_LONG) > sectorsize) {
1500 printk(KERN_WARNING
1501 "XFS: %u byte sectors in use on device %s. "
1502 "This is suboptimal; %u or greater is ideal.\n",
1503 sectorsize, XFS_BUFTARG_NAME(btp),
1504 (unsigned int)PAGE_CACHE_SIZE / BITS_PER_LONG);
1505 }
1506
1507 return 0;
1508}
1509
1510/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001511 * When allocating the initial buffer target we have not yet
1512 * read in the superblock, so don't know what sized sectors
1513 * are being used is at this early stage. Play safe.
1514 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001515STATIC int
1516xfs_setsize_buftarg_early(
1517 xfs_buftarg_t *btp,
1518 struct block_device *bdev)
1519{
1520 return xfs_setsize_buftarg_flags(btp,
1521 PAGE_CACHE_SIZE, bdev_hardsect_size(bdev), 0);
1522}
1523
1524int
1525xfs_setsize_buftarg(
1526 xfs_buftarg_t *btp,
1527 unsigned int blocksize,
1528 unsigned int sectorsize)
1529{
1530 return xfs_setsize_buftarg_flags(btp, blocksize, sectorsize, 1);
1531}
1532
1533STATIC int
1534xfs_mapping_buftarg(
1535 xfs_buftarg_t *btp,
1536 struct block_device *bdev)
1537{
1538 struct backing_dev_info *bdi;
1539 struct inode *inode;
1540 struct address_space *mapping;
Christoph Hellwigf5e54d62006-06-28 04:26:44 -07001541 static const struct address_space_operations mapping_aops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001542 .sync_page = block_sync_page,
Christoph Lametere965f962006-02-01 03:05:41 -08001543 .migratepage = fail_migrate_page,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001544 };
1545
1546 inode = new_inode(bdev->bd_inode->i_sb);
1547 if (!inode) {
1548 printk(KERN_WARNING
1549 "XFS: Cannot allocate mapping inode for device %s\n",
1550 XFS_BUFTARG_NAME(btp));
1551 return ENOMEM;
1552 }
1553 inode->i_mode = S_IFBLK;
1554 inode->i_bdev = bdev;
1555 inode->i_rdev = bdev->bd_dev;
1556 bdi = blk_get_backing_dev_info(bdev);
1557 if (!bdi)
1558 bdi = &default_backing_dev_info;
1559 mapping = &inode->i_data;
1560 mapping->a_ops = &mapping_aops;
1561 mapping->backing_dev_info = bdi;
1562 mapping_set_gfp_mask(mapping, GFP_NOFS);
Nathan Scottce8e9222006-01-11 15:39:08 +11001563 btp->bt_mapping = mapping;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001564 return 0;
1565}
1566
David Chinnera6867a62006-01-11 15:37:58 +11001567STATIC int
1568xfs_alloc_delwrite_queue(
1569 xfs_buftarg_t *btp)
1570{
1571 int error = 0;
1572
1573 INIT_LIST_HEAD(&btp->bt_list);
1574 INIT_LIST_HEAD(&btp->bt_delwrite_queue);
1575 spinlock_init(&btp->bt_delwrite_lock, "delwri_lock");
1576 btp->bt_flags = 0;
1577 btp->bt_task = kthread_run(xfsbufd, btp, "xfsbufd");
1578 if (IS_ERR(btp->bt_task)) {
1579 error = PTR_ERR(btp->bt_task);
1580 goto out_error;
1581 }
1582 xfs_register_buftarg(btp);
1583out_error:
1584 return error;
1585}
1586
Linus Torvalds1da177e2005-04-16 15:20:36 -07001587xfs_buftarg_t *
1588xfs_alloc_buftarg(
1589 struct block_device *bdev,
1590 int external)
1591{
1592 xfs_buftarg_t *btp;
1593
1594 btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
1595
Nathan Scottce8e9222006-01-11 15:39:08 +11001596 btp->bt_dev = bdev->bd_dev;
1597 btp->bt_bdev = bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001598 if (xfs_setsize_buftarg_early(btp, bdev))
1599 goto error;
1600 if (xfs_mapping_buftarg(btp, bdev))
1601 goto error;
David Chinnera6867a62006-01-11 15:37:58 +11001602 if (xfs_alloc_delwrite_queue(btp))
1603 goto error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001604 xfs_alloc_bufhash(btp, external);
1605 return btp;
1606
1607error:
1608 kmem_free(btp, sizeof(*btp));
1609 return NULL;
1610}
1611
1612
1613/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001614 * Delayed write buffer handling
Linus Torvalds1da177e2005-04-16 15:20:36 -07001615 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001616STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001617xfs_buf_delwri_queue(
1618 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001619 int unlock)
1620{
Nathan Scottce8e9222006-01-11 15:39:08 +11001621 struct list_head *dwq = &bp->b_target->bt_delwrite_queue;
1622 spinlock_t *dwlk = &bp->b_target->bt_delwrite_lock;
David Chinnera6867a62006-01-11 15:37:58 +11001623
Nathan Scottce8e9222006-01-11 15:39:08 +11001624 XB_TRACE(bp, "delwri_q", (long)unlock);
1625 ASSERT((bp->b_flags&(XBF_DELWRI|XBF_ASYNC)) == (XBF_DELWRI|XBF_ASYNC));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001626
David Chinnera6867a62006-01-11 15:37:58 +11001627 spin_lock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001628 /* If already in the queue, dequeue and place at tail */
Nathan Scottce8e9222006-01-11 15:39:08 +11001629 if (!list_empty(&bp->b_list)) {
1630 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1631 if (unlock)
1632 atomic_dec(&bp->b_hold);
1633 list_del(&bp->b_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001634 }
1635
Nathan Scottce8e9222006-01-11 15:39:08 +11001636 bp->b_flags |= _XBF_DELWRI_Q;
1637 list_add_tail(&bp->b_list, dwq);
1638 bp->b_queuetime = jiffies;
David Chinnera6867a62006-01-11 15:37:58 +11001639 spin_unlock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001640
1641 if (unlock)
Nathan Scottce8e9222006-01-11 15:39:08 +11001642 xfs_buf_unlock(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001643}
1644
1645void
Nathan Scottce8e9222006-01-11 15:39:08 +11001646xfs_buf_delwri_dequeue(
1647 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001648{
Nathan Scottce8e9222006-01-11 15:39:08 +11001649 spinlock_t *dwlk = &bp->b_target->bt_delwrite_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001650 int dequeued = 0;
1651
David Chinnera6867a62006-01-11 15:37:58 +11001652 spin_lock(dwlk);
Nathan Scottce8e9222006-01-11 15:39:08 +11001653 if ((bp->b_flags & XBF_DELWRI) && !list_empty(&bp->b_list)) {
1654 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1655 list_del_init(&bp->b_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001656 dequeued = 1;
1657 }
Nathan Scottce8e9222006-01-11 15:39:08 +11001658 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
David Chinnera6867a62006-01-11 15:37:58 +11001659 spin_unlock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001660
1661 if (dequeued)
Nathan Scottce8e9222006-01-11 15:39:08 +11001662 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001663
Nathan Scottce8e9222006-01-11 15:39:08 +11001664 XB_TRACE(bp, "delwri_dq", (long)dequeued);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001665}
1666
1667STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001668xfs_buf_runall_queues(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001669 struct workqueue_struct *queue)
1670{
1671 flush_workqueue(queue);
1672}
1673
Linus Torvalds1da177e2005-04-16 15:20:36 -07001674STATIC int
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001675xfsbufd_wakeup(
Nathan Scott15c84a42005-11-04 10:51:01 +11001676 int priority,
1677 gfp_t mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001678{
Christoph Hellwigda7f93e2006-01-11 20:49:57 +11001679 xfs_buftarg_t *btp;
David Chinnera6867a62006-01-11 15:37:58 +11001680
1681 spin_lock(&xfs_buftarg_lock);
Christoph Hellwigda7f93e2006-01-11 20:49:57 +11001682 list_for_each_entry(btp, &xfs_buftarg_list, bt_list) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001683 if (test_bit(XBT_FORCE_SLEEP, &btp->bt_flags))
David Chinnera6867a62006-01-11 15:37:58 +11001684 continue;
Nathan Scottce8e9222006-01-11 15:39:08 +11001685 set_bit(XBT_FORCE_FLUSH, &btp->bt_flags);
David Chinnera6867a62006-01-11 15:37:58 +11001686 wake_up_process(btp->bt_task);
1687 }
1688 spin_unlock(&xfs_buftarg_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001689 return 0;
1690}
1691
David Chinner585e6d82007-02-10 18:32:29 +11001692/*
1693 * Move as many buffers as specified to the supplied list
1694 * idicating if we skipped any buffers to prevent deadlocks.
1695 */
1696STATIC int
1697xfs_buf_delwri_split(
1698 xfs_buftarg_t *target,
1699 struct list_head *list,
David Chinner5e6a07d2007-02-10 18:34:49 +11001700 unsigned long age)
David Chinner585e6d82007-02-10 18:32:29 +11001701{
1702 xfs_buf_t *bp, *n;
1703 struct list_head *dwq = &target->bt_delwrite_queue;
1704 spinlock_t *dwlk = &target->bt_delwrite_lock;
1705 int skipped = 0;
David Chinner5e6a07d2007-02-10 18:34:49 +11001706 int force;
David Chinner585e6d82007-02-10 18:32:29 +11001707
David Chinner5e6a07d2007-02-10 18:34:49 +11001708 force = test_and_clear_bit(XBT_FORCE_FLUSH, &target->bt_flags);
David Chinner585e6d82007-02-10 18:32:29 +11001709 INIT_LIST_HEAD(list);
1710 spin_lock(dwlk);
1711 list_for_each_entry_safe(bp, n, dwq, b_list) {
1712 XB_TRACE(bp, "walkq1", (long)xfs_buf_ispin(bp));
1713 ASSERT(bp->b_flags & XBF_DELWRI);
1714
1715 if (!xfs_buf_ispin(bp) && !xfs_buf_cond_lock(bp)) {
David Chinner5e6a07d2007-02-10 18:34:49 +11001716 if (!force &&
David Chinner585e6d82007-02-10 18:32:29 +11001717 time_before(jiffies, bp->b_queuetime + age)) {
1718 xfs_buf_unlock(bp);
1719 break;
1720 }
1721
1722 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q|
1723 _XBF_RUN_QUEUES);
1724 bp->b_flags |= XBF_WRITE;
1725 list_move_tail(&bp->b_list, list);
1726 } else
1727 skipped++;
1728 }
1729 spin_unlock(dwlk);
1730
1731 return skipped;
1732
1733}
1734
Linus Torvalds1da177e2005-04-16 15:20:36 -07001735STATIC int
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001736xfsbufd(
David Chinner585e6d82007-02-10 18:32:29 +11001737 void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001738{
David Chinner585e6d82007-02-10 18:32:29 +11001739 struct list_head tmp;
1740 xfs_buftarg_t *target = (xfs_buftarg_t *)data;
1741 int count;
1742 xfs_buf_t *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001743
Linus Torvalds1da177e2005-04-16 15:20:36 -07001744 current->flags |= PF_MEMALLOC;
1745
Rafael J. Wysocki978c7b22007-12-07 14:09:02 +11001746 set_freezable();
1747
Linus Torvalds1da177e2005-04-16 15:20:36 -07001748 do {
Christoph Lameter3e1d1d22005-06-24 23:13:50 -07001749 if (unlikely(freezing(current))) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001750 set_bit(XBT_FORCE_SLEEP, &target->bt_flags);
Christoph Lameter3e1d1d22005-06-24 23:13:50 -07001751 refrigerator();
Nathan Scottabd0cf72005-05-05 13:30:13 -07001752 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +11001753 clear_bit(XBT_FORCE_SLEEP, &target->bt_flags);
Nathan Scottabd0cf72005-05-05 13:30:13 -07001754 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001755
Nathan Scott15c84a42005-11-04 10:51:01 +11001756 schedule_timeout_interruptible(
1757 xfs_buf_timer_centisecs * msecs_to_jiffies(10));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001758
David Chinner585e6d82007-02-10 18:32:29 +11001759 xfs_buf_delwri_split(target, &tmp,
David Chinner5e6a07d2007-02-10 18:34:49 +11001760 xfs_buf_age_centisecs * msecs_to_jiffies(10));
David Chinner585e6d82007-02-10 18:32:29 +11001761
Nathan Scottf07c2252006-09-28 10:52:15 +10001762 count = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001763 while (!list_empty(&tmp)) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001764 bp = list_entry(tmp.next, xfs_buf_t, b_list);
1765 ASSERT(target == bp->b_target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001766
Nathan Scottce8e9222006-01-11 15:39:08 +11001767 list_del_init(&bp->b_list);
1768 xfs_buf_iostrategy(bp);
David Chinner585e6d82007-02-10 18:32:29 +11001769 count++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001770 }
1771
1772 if (as_list_len > 0)
1773 purge_addresses();
Nathan Scottf07c2252006-09-28 10:52:15 +10001774 if (count)
1775 blk_run_address_space(target->bt_mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001776
Christoph Hellwig4df08c52005-09-05 08:34:18 +10001777 } while (!kthread_should_stop());
Linus Torvalds1da177e2005-04-16 15:20:36 -07001778
Christoph Hellwig4df08c52005-09-05 08:34:18 +10001779 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001780}
1781
1782/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001783 * Go through all incore buffers, and release buffers if they belong to
1784 * the given device. This is used in filesystem error handling to
1785 * preserve the consistency of its metadata.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001786 */
1787int
1788xfs_flush_buftarg(
David Chinner585e6d82007-02-10 18:32:29 +11001789 xfs_buftarg_t *target,
1790 int wait)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001791{
David Chinner585e6d82007-02-10 18:32:29 +11001792 struct list_head tmp;
1793 xfs_buf_t *bp, *n;
1794 int pincount = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001795
Nathan Scottce8e9222006-01-11 15:39:08 +11001796 xfs_buf_runall_queues(xfsdatad_workqueue);
1797 xfs_buf_runall_queues(xfslogd_workqueue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001798
David Chinner5e6a07d2007-02-10 18:34:49 +11001799 set_bit(XBT_FORCE_FLUSH, &target->bt_flags);
1800 pincount = xfs_buf_delwri_split(target, &tmp, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001801
1802 /*
1803 * Dropped the delayed write list lock, now walk the temporary list
1804 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001805 list_for_each_entry_safe(bp, n, &tmp, b_list) {
David Chinner585e6d82007-02-10 18:32:29 +11001806 ASSERT(target == bp->b_target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001807 if (wait)
Nathan Scottce8e9222006-01-11 15:39:08 +11001808 bp->b_flags &= ~XBF_ASYNC;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001809 else
Nathan Scottce8e9222006-01-11 15:39:08 +11001810 list_del_init(&bp->b_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001811
Nathan Scottce8e9222006-01-11 15:39:08 +11001812 xfs_buf_iostrategy(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001813 }
1814
Nathan Scottf07c2252006-09-28 10:52:15 +10001815 if (wait)
1816 blk_run_address_space(target->bt_mapping);
1817
Linus Torvalds1da177e2005-04-16 15:20:36 -07001818 /*
1819 * Remaining list items must be flushed before returning
1820 */
1821 while (!list_empty(&tmp)) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001822 bp = list_entry(tmp.next, xfs_buf_t, b_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001823
Nathan Scottce8e9222006-01-11 15:39:08 +11001824 list_del_init(&bp->b_list);
1825 xfs_iowait(bp);
1826 xfs_buf_relse(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001827 }
1828
Linus Torvalds1da177e2005-04-16 15:20:36 -07001829 return pincount;
1830}
1831
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001832int __init
Nathan Scottce8e9222006-01-11 15:39:08 +11001833xfs_buf_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001834{
Nathan Scottce8e9222006-01-11 15:39:08 +11001835#ifdef XFS_BUF_TRACE
1836 xfs_buf_trace_buf = ktrace_alloc(XFS_BUF_TRACE_SIZE, KM_SLEEP);
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001837#endif
1838
Nathan Scott87582802006-03-14 13:18:19 +11001839 xfs_buf_zone = kmem_zone_init_flags(sizeof(xfs_buf_t), "xfs_buf",
1840 KM_ZONE_HWALIGN, NULL);
Nathan Scottce8e9222006-01-11 15:39:08 +11001841 if (!xfs_buf_zone)
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001842 goto out_free_trace_buf;
1843
Rafael J. Wysockib4337692007-03-22 00:11:27 -08001844 xfslogd_workqueue = create_workqueue("xfslogd");
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001845 if (!xfslogd_workqueue)
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001846 goto out_free_buf_zone;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001847
Rafael J. Wysockib4337692007-03-22 00:11:27 -08001848 xfsdatad_workqueue = create_workqueue("xfsdatad");
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001849 if (!xfsdatad_workqueue)
1850 goto out_destroy_xfslogd_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001851
Rusty Russell8e1f9362007-07-17 04:03:17 -07001852 register_shrinker(&xfs_buf_shake);
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001853 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001854
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001855 out_destroy_xfslogd_workqueue:
1856 destroy_workqueue(xfslogd_workqueue);
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001857 out_free_buf_zone:
Nathan Scottce8e9222006-01-11 15:39:08 +11001858 kmem_zone_destroy(xfs_buf_zone);
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001859 out_free_trace_buf:
Nathan Scottce8e9222006-01-11 15:39:08 +11001860#ifdef XFS_BUF_TRACE
1861 ktrace_free(xfs_buf_trace_buf);
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001862#endif
Nathan Scott87582802006-03-14 13:18:19 +11001863 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001864}
1865
Linus Torvalds1da177e2005-04-16 15:20:36 -07001866void
Nathan Scottce8e9222006-01-11 15:39:08 +11001867xfs_buf_terminate(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001868{
Rusty Russell8e1f9362007-07-17 04:03:17 -07001869 unregister_shrinker(&xfs_buf_shake);
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001870 destroy_workqueue(xfsdatad_workqueue);
1871 destroy_workqueue(xfslogd_workqueue);
Nathan Scottce8e9222006-01-11 15:39:08 +11001872 kmem_zone_destroy(xfs_buf_zone);
1873#ifdef XFS_BUF_TRACE
1874 ktrace_free(xfs_buf_trace_buf);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001875#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001876}
Tim Shimmine6a0e9c2007-05-08 13:49:59 +10001877
1878#ifdef CONFIG_KDB_MODULES
1879struct list_head *
1880xfs_get_buftarg_list(void)
1881{
1882 return &xfs_buftarg_list;
1883}
1884#endif