blob: 492465c6e0b400fc7b5783c7651e00b3b6bd3bc0 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Nathan Scottf07c2252006-09-28 10:52:15 +10002 * Copyright (c) 2000-2006 Silicon Graphics, Inc.
Nathan Scott7b718762005-11-02 14:58:39 +11003 * All Rights Reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
Nathan Scott7b718762005-11-02 14:58:39 +11005 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License as
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 * published by the Free Software Foundation.
8 *
Nathan Scott7b718762005-11-02 14:58:39 +11009 * This program is distributed in the hope that it would be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 *
Nathan Scott7b718762005-11-02 14:58:39 +110014 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
Linus Torvalds1da177e2005-04-16 15:20:36 -070017 */
Vlad Apostolov93c189c2006-11-11 18:03:49 +110018#include "xfs.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/stddef.h>
20#include <linux/errno.h>
21#include <linux/slab.h>
22#include <linux/pagemap.h>
23#include <linux/init.h>
24#include <linux/vmalloc.h>
25#include <linux/bio.h>
26#include <linux/sysctl.h>
27#include <linux/proc_fs.h>
28#include <linux/workqueue.h>
29#include <linux/percpu.h>
30#include <linux/blkdev.h>
31#include <linux/hash.h>
Christoph Hellwig4df08c52005-09-05 08:34:18 +100032#include <linux/kthread.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080033#include <linux/migrate.h>
Andrew Morton3fcfab12006-10-19 23:28:16 -070034#include <linux/backing-dev.h>
Nigel Cunningham7dfb7102006-12-06 20:34:23 -080035#include <linux/freezer.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070036
Christoph Hellwigb7963132009-03-03 14:48:37 -050037#include "xfs_sb.h"
38#include "xfs_inum.h"
39#include "xfs_ag.h"
40#include "xfs_dmapi.h"
41#include "xfs_mount.h"
Christoph Hellwig0b1b2132009-12-14 23:14:59 +000042#include "xfs_trace.h"
Christoph Hellwigb7963132009-03-03 14:48:37 -050043
David Chinner7989cb82007-02-10 18:34:56 +110044static kmem_zone_t *xfs_buf_zone;
David Chinnera6867a62006-01-11 15:37:58 +110045STATIC int xfsbufd(void *);
Al Viro27496a82005-10-21 03:20:48 -040046STATIC int xfsbufd_wakeup(int, gfp_t);
Nathan Scottce8e9222006-01-11 15:39:08 +110047STATIC void xfs_buf_delwri_queue(xfs_buf_t *, int);
Rusty Russell8e1f9362007-07-17 04:03:17 -070048static struct shrinker xfs_buf_shake = {
49 .shrink = xfsbufd_wakeup,
50 .seeks = DEFAULT_SEEKS,
51};
Christoph Hellwig23ea4032005-06-21 15:14:01 +100052
David Chinner7989cb82007-02-10 18:34:56 +110053static struct workqueue_struct *xfslogd_workqueue;
Christoph Hellwig0829c362005-09-02 16:58:49 +100054struct workqueue_struct *xfsdatad_workqueue;
Dave Chinnerc626d172009-04-06 18:42:11 +020055struct workqueue_struct *xfsconvertd_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070056
Nathan Scottce8e9222006-01-11 15:39:08 +110057#ifdef XFS_BUF_LOCK_TRACKING
58# define XB_SET_OWNER(bp) ((bp)->b_last_holder = current->pid)
59# define XB_CLEAR_OWNER(bp) ((bp)->b_last_holder = -1)
60# define XB_GET_OWNER(bp) ((bp)->b_last_holder)
Linus Torvalds1da177e2005-04-16 15:20:36 -070061#else
Nathan Scottce8e9222006-01-11 15:39:08 +110062# define XB_SET_OWNER(bp) do { } while (0)
63# define XB_CLEAR_OWNER(bp) do { } while (0)
64# define XB_GET_OWNER(bp) do { } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -070065#endif
66
Nathan Scottce8e9222006-01-11 15:39:08 +110067#define xb_to_gfp(flags) \
68 ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : \
69 ((flags) & XBF_DONT_BLOCK) ? GFP_NOFS : GFP_KERNEL) | __GFP_NOWARN)
Linus Torvalds1da177e2005-04-16 15:20:36 -070070
Nathan Scottce8e9222006-01-11 15:39:08 +110071#define xb_to_km(flags) \
72 (((flags) & XBF_DONT_BLOCK) ? KM_NOFS : KM_SLEEP)
Linus Torvalds1da177e2005-04-16 15:20:36 -070073
Nathan Scottce8e9222006-01-11 15:39:08 +110074#define xfs_buf_allocate(flags) \
75 kmem_zone_alloc(xfs_buf_zone, xb_to_km(flags))
76#define xfs_buf_deallocate(bp) \
77 kmem_zone_free(xfs_buf_zone, (bp));
Linus Torvalds1da177e2005-04-16 15:20:36 -070078
79/*
Nathan Scottce8e9222006-01-11 15:39:08 +110080 * Page Region interfaces.
Linus Torvalds1da177e2005-04-16 15:20:36 -070081 *
Nathan Scottce8e9222006-01-11 15:39:08 +110082 * For pages in filesystems where the blocksize is smaller than the
83 * pagesize, we use the page->private field (long) to hold a bitmap
84 * of uptodate regions within the page.
Linus Torvalds1da177e2005-04-16 15:20:36 -070085 *
Nathan Scottce8e9222006-01-11 15:39:08 +110086 * Each such region is "bytes per page / bits per long" bytes long.
Linus Torvalds1da177e2005-04-16 15:20:36 -070087 *
Nathan Scottce8e9222006-01-11 15:39:08 +110088 * NBPPR == number-of-bytes-per-page-region
89 * BTOPR == bytes-to-page-region (rounded up)
90 * BTOPRT == bytes-to-page-region-truncated (rounded down)
Linus Torvalds1da177e2005-04-16 15:20:36 -070091 */
92#if (BITS_PER_LONG == 32)
93#define PRSHIFT (PAGE_CACHE_SHIFT - 5) /* (32 == 1<<5) */
94#elif (BITS_PER_LONG == 64)
95#define PRSHIFT (PAGE_CACHE_SHIFT - 6) /* (64 == 1<<6) */
96#else
97#error BITS_PER_LONG must be 32 or 64
98#endif
99#define NBPPR (PAGE_CACHE_SIZE/BITS_PER_LONG)
100#define BTOPR(b) (((unsigned int)(b) + (NBPPR - 1)) >> PRSHIFT)
101#define BTOPRT(b) (((unsigned int)(b) >> PRSHIFT))
102
103STATIC unsigned long
104page_region_mask(
105 size_t offset,
106 size_t length)
107{
108 unsigned long mask;
109 int first, final;
110
111 first = BTOPR(offset);
112 final = BTOPRT(offset + length - 1);
113 first = min(first, final);
114
115 mask = ~0UL;
116 mask <<= BITS_PER_LONG - (final - first);
117 mask >>= BITS_PER_LONG - (final);
118
119 ASSERT(offset + length <= PAGE_CACHE_SIZE);
120 ASSERT((final - first) < BITS_PER_LONG && (final - first) >= 0);
121
122 return mask;
123}
124
Christoph Hellwigb8f82a42009-11-14 16:17:22 +0000125STATIC void
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126set_page_region(
127 struct page *page,
128 size_t offset,
129 size_t length)
130{
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700131 set_page_private(page,
132 page_private(page) | page_region_mask(offset, length));
133 if (page_private(page) == ~0UL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134 SetPageUptodate(page);
135}
136
Christoph Hellwigb8f82a42009-11-14 16:17:22 +0000137STATIC int
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138test_page_region(
139 struct page *page,
140 size_t offset,
141 size_t length)
142{
143 unsigned long mask = page_region_mask(offset, length);
144
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700145 return (mask && (page_private(page) & mask) == mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146}
147
148/*
Felix Blyakher3a011a12009-02-18 15:56:51 -0600149 * Mapping of multi-page buffers into contiguous virtual space
150 */
151
152typedef struct a_list {
153 void *vm_addr;
154 struct a_list *next;
155} a_list_t;
156
157static a_list_t *as_free_head;
158static int as_list_len;
159static DEFINE_SPINLOCK(as_lock);
160
161/*
162 * Try to batch vunmaps because they are costly.
163 */
164STATIC void
165free_address(
166 void *addr)
167{
168 a_list_t *aentry;
169
170#ifdef CONFIG_XEN
171 /*
172 * Xen needs to be able to make sure it can get an exclusive
173 * RO mapping of pages it wants to turn into a pagetable. If
174 * a newly allocated page is also still being vmap()ed by xfs,
175 * it will cause pagetable construction to fail. This is a
176 * quick workaround to always eagerly unmap pages so that Xen
177 * is happy.
178 */
179 vunmap(addr);
180 return;
181#endif
182
183 aentry = kmalloc(sizeof(a_list_t), GFP_NOWAIT);
184 if (likely(aentry)) {
185 spin_lock(&as_lock);
186 aentry->next = as_free_head;
187 aentry->vm_addr = addr;
188 as_free_head = aentry;
189 as_list_len++;
190 spin_unlock(&as_lock);
191 } else {
192 vunmap(addr);
193 }
194}
195
196STATIC void
197purge_addresses(void)
198{
199 a_list_t *aentry, *old;
200
201 if (as_free_head == NULL)
202 return;
203
204 spin_lock(&as_lock);
205 aentry = as_free_head;
206 as_free_head = NULL;
207 as_list_len = 0;
208 spin_unlock(&as_lock);
209
210 while ((old = aentry) != NULL) {
211 vunmap(aentry->vm_addr);
212 aentry = aentry->next;
213 kfree(old);
214 }
215}
216
217/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100218 * Internal xfs_buf_t object manipulation
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219 */
220
221STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +1100222_xfs_buf_initialize(
223 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224 xfs_buftarg_t *target,
Nathan Scott204ab252006-01-11 20:50:22 +1100225 xfs_off_t range_base,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700226 size_t range_length,
Nathan Scottce8e9222006-01-11 15:39:08 +1100227 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228{
229 /*
Nathan Scottce8e9222006-01-11 15:39:08 +1100230 * We don't want certain flags to appear in b_flags.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700231 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100232 flags &= ~(XBF_LOCK|XBF_MAPPED|XBF_DONT_BLOCK|XBF_READ_AHEAD);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233
Nathan Scottce8e9222006-01-11 15:39:08 +1100234 memset(bp, 0, sizeof(xfs_buf_t));
235 atomic_set(&bp->b_hold, 1);
David Chinnerb4dd3302008-08-13 16:36:11 +1000236 init_completion(&bp->b_iowait);
Nathan Scottce8e9222006-01-11 15:39:08 +1100237 INIT_LIST_HEAD(&bp->b_list);
238 INIT_LIST_HEAD(&bp->b_hash_list);
239 init_MUTEX_LOCKED(&bp->b_sema); /* held, no waiters */
240 XB_SET_OWNER(bp);
241 bp->b_target = target;
242 bp->b_file_offset = range_base;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243 /*
244 * Set buffer_length and count_desired to the same value initially.
245 * I/O routines should use count_desired, which will be the same in
246 * most cases but may be reset (e.g. XFS recovery).
247 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100248 bp->b_buffer_length = bp->b_count_desired = range_length;
249 bp->b_flags = flags;
250 bp->b_bn = XFS_BUF_DADDR_NULL;
251 atomic_set(&bp->b_pin_count, 0);
252 init_waitqueue_head(&bp->b_waiters);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700253
Nathan Scottce8e9222006-01-11 15:39:08 +1100254 XFS_STATS_INC(xb_create);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000255
256 trace_xfs_buf_init(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700257}
258
259/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100260 * Allocate a page array capable of holding a specified number
261 * of pages, and point the page buf at it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700262 */
263STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100264_xfs_buf_get_pages(
265 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266 int page_count,
Nathan Scottce8e9222006-01-11 15:39:08 +1100267 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268{
269 /* Make sure that we have a page list */
Nathan Scottce8e9222006-01-11 15:39:08 +1100270 if (bp->b_pages == NULL) {
271 bp->b_offset = xfs_buf_poff(bp->b_file_offset);
272 bp->b_page_count = page_count;
273 if (page_count <= XB_PAGES) {
274 bp->b_pages = bp->b_page_array;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100276 bp->b_pages = kmem_alloc(sizeof(struct page *) *
277 page_count, xb_to_km(flags));
278 if (bp->b_pages == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279 return -ENOMEM;
280 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100281 memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282 }
283 return 0;
284}
285
286/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100287 * Frees b_pages if it was allocated.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700288 */
289STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +1100290_xfs_buf_free_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291 xfs_buf_t *bp)
292{
Nathan Scottce8e9222006-01-11 15:39:08 +1100293 if (bp->b_pages != bp->b_page_array) {
Denys Vlasenkof0e2d932008-05-19 16:31:57 +1000294 kmem_free(bp->b_pages);
Dave Chinner3fc98b12009-12-14 23:11:57 +0000295 bp->b_pages = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700296 }
297}
298
299/*
300 * Releases the specified buffer.
301 *
302 * The modification state of any associated pages is left unchanged.
Nathan Scottce8e9222006-01-11 15:39:08 +1100303 * The buffer most not be on any hash - use xfs_buf_rele instead for
Linus Torvalds1da177e2005-04-16 15:20:36 -0700304 * hashed and refcounted buffers
305 */
306void
Nathan Scottce8e9222006-01-11 15:39:08 +1100307xfs_buf_free(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308 xfs_buf_t *bp)
309{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000310 trace_xfs_buf_free(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311
Nathan Scottce8e9222006-01-11 15:39:08 +1100312 ASSERT(list_empty(&bp->b_hash_list));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000314 if (bp->b_flags & (_XBF_PAGE_CACHE|_XBF_PAGES)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700315 uint i;
316
Nathan Scottce8e9222006-01-11 15:39:08 +1100317 if ((bp->b_flags & XBF_MAPPED) && (bp->b_page_count > 1))
Felix Blyakher3a011a12009-02-18 15:56:51 -0600318 free_address(bp->b_addr - bp->b_offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700319
Nathan Scott948ecdb2006-09-28 11:03:13 +1000320 for (i = 0; i < bp->b_page_count; i++) {
321 struct page *page = bp->b_pages[i];
322
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000323 if (bp->b_flags & _XBF_PAGE_CACHE)
324 ASSERT(!PagePrivate(page));
Nathan Scott948ecdb2006-09-28 11:03:13 +1000325 page_cache_release(page);
326 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700327 }
Dave Chinner3fc98b12009-12-14 23:11:57 +0000328 _xfs_buf_free_pages(bp);
Nathan Scottce8e9222006-01-11 15:39:08 +1100329 xfs_buf_deallocate(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700330}
331
332/*
333 * Finds all pages for buffer in question and builds it's page list.
334 */
335STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100336_xfs_buf_lookup_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700337 xfs_buf_t *bp,
338 uint flags)
339{
Nathan Scottce8e9222006-01-11 15:39:08 +1100340 struct address_space *mapping = bp->b_target->bt_mapping;
341 size_t blocksize = bp->b_target->bt_bsize;
342 size_t size = bp->b_count_desired;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343 size_t nbytes, offset;
Nathan Scottce8e9222006-01-11 15:39:08 +1100344 gfp_t gfp_mask = xb_to_gfp(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700345 unsigned short page_count, i;
346 pgoff_t first;
Nathan Scott204ab252006-01-11 20:50:22 +1100347 xfs_off_t end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348 int error;
349
Nathan Scottce8e9222006-01-11 15:39:08 +1100350 end = bp->b_file_offset + bp->b_buffer_length;
351 page_count = xfs_buf_btoc(end) - xfs_buf_btoct(bp->b_file_offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700352
Nathan Scottce8e9222006-01-11 15:39:08 +1100353 error = _xfs_buf_get_pages(bp, page_count, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700354 if (unlikely(error))
355 return error;
Nathan Scottce8e9222006-01-11 15:39:08 +1100356 bp->b_flags |= _XBF_PAGE_CACHE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357
Nathan Scottce8e9222006-01-11 15:39:08 +1100358 offset = bp->b_offset;
359 first = bp->b_file_offset >> PAGE_CACHE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360
Nathan Scottce8e9222006-01-11 15:39:08 +1100361 for (i = 0; i < bp->b_page_count; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362 struct page *page;
363 uint retries = 0;
364
365 retry:
366 page = find_or_create_page(mapping, first + i, gfp_mask);
367 if (unlikely(page == NULL)) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100368 if (flags & XBF_READ_AHEAD) {
369 bp->b_page_count = i;
Christoph Hellwig6ab455e2008-05-19 16:34:42 +1000370 for (i = 0; i < bp->b_page_count; i++)
371 unlock_page(bp->b_pages[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700372 return -ENOMEM;
373 }
374
375 /*
376 * This could deadlock.
377 *
378 * But until all the XFS lowlevel code is revamped to
379 * handle buffer allocation failures we can't do much.
380 */
381 if (!(++retries % 100))
382 printk(KERN_ERR
383 "XFS: possible memory allocation "
384 "deadlock in %s (mode:0x%x)\n",
Harvey Harrison34a622b2008-04-10 12:19:21 +1000385 __func__, gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386
Nathan Scottce8e9222006-01-11 15:39:08 +1100387 XFS_STATS_INC(xb_page_retries);
Christoph Hellwig23ea4032005-06-21 15:14:01 +1000388 xfsbufd_wakeup(0, gfp_mask);
Jens Axboe8aa7e842009-07-09 14:52:32 +0200389 congestion_wait(BLK_RW_ASYNC, HZ/50);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390 goto retry;
391 }
392
Nathan Scottce8e9222006-01-11 15:39:08 +1100393 XFS_STATS_INC(xb_page_found);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394
395 nbytes = min_t(size_t, size, PAGE_CACHE_SIZE - offset);
396 size -= nbytes;
397
Nathan Scott948ecdb2006-09-28 11:03:13 +1000398 ASSERT(!PagePrivate(page));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700399 if (!PageUptodate(page)) {
400 page_count--;
Christoph Hellwig6ab455e2008-05-19 16:34:42 +1000401 if (blocksize >= PAGE_CACHE_SIZE) {
402 if (flags & XBF_READ)
403 bp->b_flags |= _XBF_PAGE_LOCKED;
404 } else if (!PagePrivate(page)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700405 if (test_page_region(page, offset, nbytes))
406 page_count++;
407 }
408 }
409
Nathan Scottce8e9222006-01-11 15:39:08 +1100410 bp->b_pages[i] = page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411 offset = 0;
412 }
413
Christoph Hellwig6ab455e2008-05-19 16:34:42 +1000414 if (!(bp->b_flags & _XBF_PAGE_LOCKED)) {
415 for (i = 0; i < bp->b_page_count; i++)
416 unlock_page(bp->b_pages[i]);
417 }
418
Nathan Scottce8e9222006-01-11 15:39:08 +1100419 if (page_count == bp->b_page_count)
420 bp->b_flags |= XBF_DONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700421
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422 return error;
423}
424
425/*
426 * Map buffer into kernel address-space if nessecary.
427 */
428STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100429_xfs_buf_map_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700430 xfs_buf_t *bp,
431 uint flags)
432{
433 /* A single page buffer is always mappable */
Nathan Scottce8e9222006-01-11 15:39:08 +1100434 if (bp->b_page_count == 1) {
435 bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
436 bp->b_flags |= XBF_MAPPED;
437 } else if (flags & XBF_MAPPED) {
Felix Blyakher3a011a12009-02-18 15:56:51 -0600438 if (as_list_len > 64)
439 purge_addresses();
Felix Blyakhercf7dab82009-02-18 15:41:28 -0600440 bp->b_addr = vmap(bp->b_pages, bp->b_page_count,
441 VM_MAP, PAGE_KERNEL);
Nathan Scottce8e9222006-01-11 15:39:08 +1100442 if (unlikely(bp->b_addr == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443 return -ENOMEM;
Nathan Scottce8e9222006-01-11 15:39:08 +1100444 bp->b_addr += bp->b_offset;
445 bp->b_flags |= XBF_MAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700446 }
447
448 return 0;
449}
450
451/*
452 * Finding and Reading Buffers
453 */
454
455/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100456 * Look up, and creates if absent, a lockable buffer for
Linus Torvalds1da177e2005-04-16 15:20:36 -0700457 * a given range of an inode. The buffer is returned
458 * locked. If other overlapping buffers exist, they are
459 * released before the new buffer is created and locked,
460 * which may imply that this call will block until those buffers
461 * are unlocked. No I/O is implied by this call.
462 */
463xfs_buf_t *
Nathan Scottce8e9222006-01-11 15:39:08 +1100464_xfs_buf_find(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465 xfs_buftarg_t *btp, /* block device target */
Nathan Scott204ab252006-01-11 20:50:22 +1100466 xfs_off_t ioff, /* starting offset of range */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467 size_t isize, /* length of range */
Nathan Scottce8e9222006-01-11 15:39:08 +1100468 xfs_buf_flags_t flags,
469 xfs_buf_t *new_bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700470{
Nathan Scott204ab252006-01-11 20:50:22 +1100471 xfs_off_t range_base;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472 size_t range_length;
473 xfs_bufhash_t *hash;
Nathan Scottce8e9222006-01-11 15:39:08 +1100474 xfs_buf_t *bp, *n;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700475
476 range_base = (ioff << BBSHIFT);
477 range_length = (isize << BBSHIFT);
478
479 /* Check for IOs smaller than the sector size / not sector aligned */
Nathan Scottce8e9222006-01-11 15:39:08 +1100480 ASSERT(!(range_length < (1 << btp->bt_sshift)));
Nathan Scott204ab252006-01-11 20:50:22 +1100481 ASSERT(!(range_base & (xfs_off_t)btp->bt_smask));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700482
483 hash = &btp->bt_hash[hash_long((unsigned long)ioff, btp->bt_hashshift)];
484
485 spin_lock(&hash->bh_lock);
486
Nathan Scottce8e9222006-01-11 15:39:08 +1100487 list_for_each_entry_safe(bp, n, &hash->bh_list, b_hash_list) {
488 ASSERT(btp == bp->b_target);
489 if (bp->b_file_offset == range_base &&
490 bp->b_buffer_length == range_length) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700491 /*
Nathan Scottce8e9222006-01-11 15:39:08 +1100492 * If we look at something, bring it to the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700493 * front of the list for next time.
494 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100495 atomic_inc(&bp->b_hold);
496 list_move(&bp->b_hash_list, &hash->bh_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497 goto found;
498 }
499 }
500
501 /* No match found */
Nathan Scottce8e9222006-01-11 15:39:08 +1100502 if (new_bp) {
503 _xfs_buf_initialize(new_bp, btp, range_base,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504 range_length, flags);
Nathan Scottce8e9222006-01-11 15:39:08 +1100505 new_bp->b_hash = hash;
506 list_add(&new_bp->b_hash_list, &hash->bh_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100508 XFS_STATS_INC(xb_miss_locked);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700509 }
510
511 spin_unlock(&hash->bh_lock);
Nathan Scottce8e9222006-01-11 15:39:08 +1100512 return new_bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513
514found:
515 spin_unlock(&hash->bh_lock);
516
517 /* Attempt to get the semaphore without sleeping,
518 * if this does not work then we need to drop the
519 * spinlock and do a hard attempt on the semaphore.
520 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100521 if (down_trylock(&bp->b_sema)) {
522 if (!(flags & XBF_TRYLOCK)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523 /* wait for buffer ownership */
Nathan Scottce8e9222006-01-11 15:39:08 +1100524 xfs_buf_lock(bp);
525 XFS_STATS_INC(xb_get_locked_waited);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526 } else {
527 /* We asked for a trylock and failed, no need
528 * to look at file offset and length here, we
Nathan Scottce8e9222006-01-11 15:39:08 +1100529 * know that this buffer at least overlaps our
530 * buffer and is locked, therefore our buffer
531 * either does not exist, or is this buffer.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700532 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100533 xfs_buf_rele(bp);
534 XFS_STATS_INC(xb_busy_locked);
535 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700536 }
537 } else {
538 /* trylock worked */
Nathan Scottce8e9222006-01-11 15:39:08 +1100539 XB_SET_OWNER(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700540 }
541
Nathan Scottce8e9222006-01-11 15:39:08 +1100542 if (bp->b_flags & XBF_STALE) {
543 ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
544 bp->b_flags &= XBF_MAPPED;
David Chinner2f926582005-09-05 08:33:35 +1000545 }
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000546
547 trace_xfs_buf_find(bp, flags, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +1100548 XFS_STATS_INC(xb_get_locked);
549 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700550}
551
552/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100553 * Assembles a buffer covering the specified range.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554 * Storage in memory for all portions of the buffer will be allocated,
555 * although backing storage may not be.
556 */
557xfs_buf_t *
Christoph Hellwig6ad112b2009-11-24 18:02:23 +0000558xfs_buf_get(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700559 xfs_buftarg_t *target,/* target for buffer */
Nathan Scott204ab252006-01-11 20:50:22 +1100560 xfs_off_t ioff, /* starting offset of range */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700561 size_t isize, /* length of range */
Nathan Scottce8e9222006-01-11 15:39:08 +1100562 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563{
Nathan Scottce8e9222006-01-11 15:39:08 +1100564 xfs_buf_t *bp, *new_bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565 int error = 0, i;
566
Nathan Scottce8e9222006-01-11 15:39:08 +1100567 new_bp = xfs_buf_allocate(flags);
568 if (unlikely(!new_bp))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569 return NULL;
570
Nathan Scottce8e9222006-01-11 15:39:08 +1100571 bp = _xfs_buf_find(target, ioff, isize, flags, new_bp);
572 if (bp == new_bp) {
573 error = _xfs_buf_lookup_pages(bp, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574 if (error)
575 goto no_buffer;
576 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100577 xfs_buf_deallocate(new_bp);
578 if (unlikely(bp == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700579 return NULL;
580 }
581
Nathan Scottce8e9222006-01-11 15:39:08 +1100582 for (i = 0; i < bp->b_page_count; i++)
583 mark_page_accessed(bp->b_pages[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584
Nathan Scottce8e9222006-01-11 15:39:08 +1100585 if (!(bp->b_flags & XBF_MAPPED)) {
586 error = _xfs_buf_map_pages(bp, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700587 if (unlikely(error)) {
588 printk(KERN_WARNING "%s: failed to map pages\n",
Harvey Harrison34a622b2008-04-10 12:19:21 +1000589 __func__);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590 goto no_buffer;
591 }
592 }
593
Nathan Scottce8e9222006-01-11 15:39:08 +1100594 XFS_STATS_INC(xb_get);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595
596 /*
597 * Always fill in the block number now, the mapped cases can do
598 * their own overlay of this later.
599 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100600 bp->b_bn = ioff;
601 bp->b_count_desired = bp->b_buffer_length;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700602
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000603 trace_xfs_buf_get(bp, flags, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +1100604 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605
606 no_buffer:
Nathan Scottce8e9222006-01-11 15:39:08 +1100607 if (flags & (XBF_LOCK | XBF_TRYLOCK))
608 xfs_buf_unlock(bp);
609 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700610 return NULL;
611}
612
Christoph Hellwig5d765b92008-12-03 12:20:26 +0100613STATIC int
614_xfs_buf_read(
615 xfs_buf_t *bp,
616 xfs_buf_flags_t flags)
617{
618 int status;
619
Christoph Hellwig5d765b92008-12-03 12:20:26 +0100620 ASSERT(!(flags & (XBF_DELWRI|XBF_WRITE)));
621 ASSERT(bp->b_bn != XFS_BUF_DADDR_NULL);
622
623 bp->b_flags &= ~(XBF_WRITE | XBF_ASYNC | XBF_DELWRI | \
624 XBF_READ_AHEAD | _XBF_RUN_QUEUES);
625 bp->b_flags |= flags & (XBF_READ | XBF_ASYNC | \
626 XBF_READ_AHEAD | _XBF_RUN_QUEUES);
627
628 status = xfs_buf_iorequest(bp);
629 if (!status && !(flags & XBF_ASYNC))
630 status = xfs_buf_iowait(bp);
631 return status;
632}
633
Linus Torvalds1da177e2005-04-16 15:20:36 -0700634xfs_buf_t *
Christoph Hellwig6ad112b2009-11-24 18:02:23 +0000635xfs_buf_read(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700636 xfs_buftarg_t *target,
Nathan Scott204ab252006-01-11 20:50:22 +1100637 xfs_off_t ioff,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638 size_t isize,
Nathan Scottce8e9222006-01-11 15:39:08 +1100639 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700640{
Nathan Scottce8e9222006-01-11 15:39:08 +1100641 xfs_buf_t *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700642
Nathan Scottce8e9222006-01-11 15:39:08 +1100643 flags |= XBF_READ;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644
Christoph Hellwig6ad112b2009-11-24 18:02:23 +0000645 bp = xfs_buf_get(target, ioff, isize, flags);
Nathan Scottce8e9222006-01-11 15:39:08 +1100646 if (bp) {
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000647 trace_xfs_buf_read(bp, flags, _RET_IP_);
648
Nathan Scottce8e9222006-01-11 15:39:08 +1100649 if (!XFS_BUF_ISDONE(bp)) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100650 XFS_STATS_INC(xb_get_read);
Christoph Hellwig5d765b92008-12-03 12:20:26 +0100651 _xfs_buf_read(bp, flags);
Nathan Scottce8e9222006-01-11 15:39:08 +1100652 } else if (flags & XBF_ASYNC) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700653 /*
654 * Read ahead call which is already satisfied,
655 * drop the buffer
656 */
657 goto no_buffer;
658 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700659 /* We do not want read in the flags */
Nathan Scottce8e9222006-01-11 15:39:08 +1100660 bp->b_flags &= ~XBF_READ;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700661 }
662 }
663
Nathan Scottce8e9222006-01-11 15:39:08 +1100664 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700665
666 no_buffer:
Nathan Scottce8e9222006-01-11 15:39:08 +1100667 if (flags & (XBF_LOCK | XBF_TRYLOCK))
668 xfs_buf_unlock(bp);
669 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700670 return NULL;
671}
672
673/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100674 * If we are not low on memory then do the readahead in a deadlock
675 * safe manner.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700676 */
677void
Nathan Scottce8e9222006-01-11 15:39:08 +1100678xfs_buf_readahead(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700679 xfs_buftarg_t *target,
Nathan Scott204ab252006-01-11 20:50:22 +1100680 xfs_off_t ioff,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700681 size_t isize,
Nathan Scottce8e9222006-01-11 15:39:08 +1100682 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683{
684 struct backing_dev_info *bdi;
685
Nathan Scottce8e9222006-01-11 15:39:08 +1100686 bdi = target->bt_mapping->backing_dev_info;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700687 if (bdi_read_congested(bdi))
688 return;
689
Nathan Scottce8e9222006-01-11 15:39:08 +1100690 flags |= (XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD);
Christoph Hellwig6ad112b2009-11-24 18:02:23 +0000691 xfs_buf_read(target, ioff, isize, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700692}
693
694xfs_buf_t *
Nathan Scottce8e9222006-01-11 15:39:08 +1100695xfs_buf_get_empty(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700696 size_t len,
697 xfs_buftarg_t *target)
698{
Nathan Scottce8e9222006-01-11 15:39:08 +1100699 xfs_buf_t *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700700
Nathan Scottce8e9222006-01-11 15:39:08 +1100701 bp = xfs_buf_allocate(0);
702 if (bp)
703 _xfs_buf_initialize(bp, target, 0, len, 0);
704 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705}
706
707static inline struct page *
708mem_to_page(
709 void *addr)
710{
Christoph Lameter9e2779f2008-02-04 22:28:34 -0800711 if ((!is_vmalloc_addr(addr))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700712 return virt_to_page(addr);
713 } else {
714 return vmalloc_to_page(addr);
715 }
716}
717
718int
Nathan Scottce8e9222006-01-11 15:39:08 +1100719xfs_buf_associate_memory(
720 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700721 void *mem,
722 size_t len)
723{
724 int rval;
725 int i = 0;
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100726 unsigned long pageaddr;
727 unsigned long offset;
728 size_t buflen;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700729 int page_count;
730
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100731 pageaddr = (unsigned long)mem & PAGE_CACHE_MASK;
732 offset = (unsigned long)mem - pageaddr;
733 buflen = PAGE_CACHE_ALIGN(len + offset);
734 page_count = buflen >> PAGE_CACHE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735
736 /* Free any previous set of page pointers */
Nathan Scottce8e9222006-01-11 15:39:08 +1100737 if (bp->b_pages)
738 _xfs_buf_free_pages(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739
Nathan Scottce8e9222006-01-11 15:39:08 +1100740 bp->b_pages = NULL;
741 bp->b_addr = mem;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700742
Christoph Hellwig36fae172009-07-18 18:14:58 -0400743 rval = _xfs_buf_get_pages(bp, page_count, XBF_DONT_BLOCK);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700744 if (rval)
745 return rval;
746
Nathan Scottce8e9222006-01-11 15:39:08 +1100747 bp->b_offset = offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700748
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100749 for (i = 0; i < bp->b_page_count; i++) {
750 bp->b_pages[i] = mem_to_page((void *)pageaddr);
751 pageaddr += PAGE_CACHE_SIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700752 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700753
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100754 bp->b_count_desired = len;
755 bp->b_buffer_length = buflen;
Nathan Scottce8e9222006-01-11 15:39:08 +1100756 bp->b_flags |= XBF_MAPPED;
Christoph Hellwig6ab455e2008-05-19 16:34:42 +1000757 bp->b_flags &= ~_XBF_PAGE_LOCKED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700758
759 return 0;
760}
761
762xfs_buf_t *
Nathan Scottce8e9222006-01-11 15:39:08 +1100763xfs_buf_get_noaddr(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700764 size_t len,
765 xfs_buftarg_t *target)
766{
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000767 unsigned long page_count = PAGE_ALIGN(len) >> PAGE_SHIFT;
768 int error, i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700769 xfs_buf_t *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700770
Nathan Scottce8e9222006-01-11 15:39:08 +1100771 bp = xfs_buf_allocate(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700772 if (unlikely(bp == NULL))
773 goto fail;
Nathan Scottce8e9222006-01-11 15:39:08 +1100774 _xfs_buf_initialize(bp, target, 0, len, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700775
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000776 error = _xfs_buf_get_pages(bp, page_count, 0);
777 if (error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700778 goto fail_free_buf;
779
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000780 for (i = 0; i < page_count; i++) {
781 bp->b_pages[i] = alloc_page(GFP_KERNEL);
782 if (!bp->b_pages[i])
783 goto fail_free_mem;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700784 }
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000785 bp->b_flags |= _XBF_PAGES;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700786
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000787 error = _xfs_buf_map_pages(bp, XBF_MAPPED);
788 if (unlikely(error)) {
789 printk(KERN_WARNING "%s: failed to map pages\n",
Harvey Harrison34a622b2008-04-10 12:19:21 +1000790 __func__);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700791 goto fail_free_mem;
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000792 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700793
Nathan Scottce8e9222006-01-11 15:39:08 +1100794 xfs_buf_unlock(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700795
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000796 trace_xfs_buf_get_noaddr(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797 return bp;
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000798
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799 fail_free_mem:
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000800 while (--i >= 0)
801 __free_page(bp->b_pages[i]);
Christoph Hellwigca165b82007-05-24 15:21:11 +1000802 _xfs_buf_free_pages(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700803 fail_free_buf:
Christoph Hellwigca165b82007-05-24 15:21:11 +1000804 xfs_buf_deallocate(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700805 fail:
806 return NULL;
807}
808
809/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700810 * Increment reference count on buffer, to hold the buffer concurrently
811 * with another thread which may release (free) the buffer asynchronously.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812 * Must hold the buffer already to call this function.
813 */
814void
Nathan Scottce8e9222006-01-11 15:39:08 +1100815xfs_buf_hold(
816 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700817{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000818 trace_xfs_buf_hold(bp, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +1100819 atomic_inc(&bp->b_hold);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700820}
821
822/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100823 * Releases a hold on the specified buffer. If the
824 * the hold count is 1, calls xfs_buf_free.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700825 */
826void
Nathan Scottce8e9222006-01-11 15:39:08 +1100827xfs_buf_rele(
828 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829{
Nathan Scottce8e9222006-01-11 15:39:08 +1100830 xfs_bufhash_t *hash = bp->b_hash;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700831
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000832 trace_xfs_buf_rele(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700833
Nathan Scottfad3aa12006-02-01 12:14:52 +1100834 if (unlikely(!hash)) {
835 ASSERT(!bp->b_relse);
836 if (atomic_dec_and_test(&bp->b_hold))
837 xfs_buf_free(bp);
838 return;
839 }
840
Lachlan McIlroy37906892008-08-13 15:42:10 +1000841 ASSERT(atomic_read(&bp->b_hold) > 0);
Nathan Scottce8e9222006-01-11 15:39:08 +1100842 if (atomic_dec_and_lock(&bp->b_hold, &hash->bh_lock)) {
843 if (bp->b_relse) {
844 atomic_inc(&bp->b_hold);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700845 spin_unlock(&hash->bh_lock);
Nathan Scottce8e9222006-01-11 15:39:08 +1100846 (*(bp->b_relse)) (bp);
847 } else if (bp->b_flags & XBF_FS_MANAGED) {
Christoph Hellwig7f14d0a2005-11-02 15:09:35 +1100848 spin_unlock(&hash->bh_lock);
849 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100850 ASSERT(!(bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)));
851 list_del_init(&bp->b_hash_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700852 spin_unlock(&hash->bh_lock);
Nathan Scottce8e9222006-01-11 15:39:08 +1100853 xfs_buf_free(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700854 }
855 }
856}
857
858
859/*
860 * Mutual exclusion on buffers. Locking model:
861 *
862 * Buffers associated with inodes for which buffer locking
863 * is not enabled are not protected by semaphores, and are
864 * assumed to be exclusively owned by the caller. There is a
865 * spinlock in the buffer, used by the caller when concurrent
866 * access is possible.
867 */
868
869/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100870 * Locks a buffer object, if it is not already locked.
871 * Note that this in no way locks the underlying pages, so it is only
872 * useful for synchronizing concurrent use of buffer objects, not for
873 * synchronizing independent access to the underlying pages.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700874 */
875int
Nathan Scottce8e9222006-01-11 15:39:08 +1100876xfs_buf_cond_lock(
877 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700878{
879 int locked;
880
Nathan Scottce8e9222006-01-11 15:39:08 +1100881 locked = down_trylock(&bp->b_sema) == 0;
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000882 if (locked)
Nathan Scottce8e9222006-01-11 15:39:08 +1100883 XB_SET_OWNER(bp);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000884
885 trace_xfs_buf_cond_lock(bp, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +1100886 return locked ? 0 : -EBUSY;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700887}
888
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889int
Nathan Scottce8e9222006-01-11 15:39:08 +1100890xfs_buf_lock_value(
891 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700892{
Stephen Rothwelladaa6932008-04-22 15:26:13 +1000893 return bp->b_sema.count;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700894}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700895
896/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100897 * Locks a buffer object.
898 * Note that this in no way locks the underlying pages, so it is only
899 * useful for synchronizing concurrent use of buffer objects, not for
900 * synchronizing independent access to the underlying pages.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100902void
903xfs_buf_lock(
904 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700905{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000906 trace_xfs_buf_lock(bp, _RET_IP_);
907
Nathan Scottce8e9222006-01-11 15:39:08 +1100908 if (atomic_read(&bp->b_io_remaining))
909 blk_run_address_space(bp->b_target->bt_mapping);
910 down(&bp->b_sema);
911 XB_SET_OWNER(bp);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000912
913 trace_xfs_buf_lock_done(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914}
915
916/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100917 * Releases the lock on the buffer object.
David Chinner2f926582005-09-05 08:33:35 +1000918 * If the buffer is marked delwri but is not queued, do so before we
Nathan Scottce8e9222006-01-11 15:39:08 +1100919 * unlock the buffer as we need to set flags correctly. We also need to
David Chinner2f926582005-09-05 08:33:35 +1000920 * take a reference for the delwri queue because the unlocker is going to
921 * drop their's and they don't know we just queued it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700922 */
923void
Nathan Scottce8e9222006-01-11 15:39:08 +1100924xfs_buf_unlock(
925 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700926{
Nathan Scottce8e9222006-01-11 15:39:08 +1100927 if ((bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)) == XBF_DELWRI) {
928 atomic_inc(&bp->b_hold);
929 bp->b_flags |= XBF_ASYNC;
930 xfs_buf_delwri_queue(bp, 0);
David Chinner2f926582005-09-05 08:33:35 +1000931 }
932
Nathan Scottce8e9222006-01-11 15:39:08 +1100933 XB_CLEAR_OWNER(bp);
934 up(&bp->b_sema);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000935
936 trace_xfs_buf_unlock(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700937}
938
939
940/*
941 * Pinning Buffer Storage in Memory
Nathan Scottce8e9222006-01-11 15:39:08 +1100942 * Ensure that no attempt to force a buffer to disk will succeed.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700943 */
944void
Nathan Scottce8e9222006-01-11 15:39:08 +1100945xfs_buf_pin(
946 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700947{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000948 trace_xfs_buf_pin(bp, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +1100949 atomic_inc(&bp->b_pin_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950}
951
Linus Torvalds1da177e2005-04-16 15:20:36 -0700952void
Nathan Scottce8e9222006-01-11 15:39:08 +1100953xfs_buf_unpin(
954 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700955{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000956 trace_xfs_buf_unpin(bp, _RET_IP_);
957
Nathan Scottce8e9222006-01-11 15:39:08 +1100958 if (atomic_dec_and_test(&bp->b_pin_count))
959 wake_up_all(&bp->b_waiters);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700960}
961
962int
Nathan Scottce8e9222006-01-11 15:39:08 +1100963xfs_buf_ispin(
964 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700965{
Nathan Scottce8e9222006-01-11 15:39:08 +1100966 return atomic_read(&bp->b_pin_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700967}
968
Nathan Scottce8e9222006-01-11 15:39:08 +1100969STATIC void
970xfs_buf_wait_unpin(
971 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700972{
973 DECLARE_WAITQUEUE (wait, current);
974
Nathan Scottce8e9222006-01-11 15:39:08 +1100975 if (atomic_read(&bp->b_pin_count) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700976 return;
977
Nathan Scottce8e9222006-01-11 15:39:08 +1100978 add_wait_queue(&bp->b_waiters, &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700979 for (;;) {
980 set_current_state(TASK_UNINTERRUPTIBLE);
Nathan Scottce8e9222006-01-11 15:39:08 +1100981 if (atomic_read(&bp->b_pin_count) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982 break;
Nathan Scottce8e9222006-01-11 15:39:08 +1100983 if (atomic_read(&bp->b_io_remaining))
984 blk_run_address_space(bp->b_target->bt_mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700985 schedule();
986 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100987 remove_wait_queue(&bp->b_waiters, &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700988 set_current_state(TASK_RUNNING);
989}
990
991/*
992 * Buffer Utility Routines
993 */
994
Linus Torvalds1da177e2005-04-16 15:20:36 -0700995STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +1100996xfs_buf_iodone_work(
David Howellsc4028952006-11-22 14:57:56 +0000997 struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700998{
David Howellsc4028952006-11-22 14:57:56 +0000999 xfs_buf_t *bp =
1000 container_of(work, xfs_buf_t, b_iodone_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001001
David Chinner0bfefc42007-05-14 18:24:23 +10001002 /*
1003 * We can get an EOPNOTSUPP to ordered writes. Here we clear the
1004 * ordered flag and reissue them. Because we can't tell the higher
1005 * layers directly that they should not issue ordered I/O anymore, they
Christoph Hellwig73f6aa42008-10-10 17:28:29 +11001006 * need to check if the _XFS_BARRIER_FAILED flag was set during I/O completion.
David Chinner0bfefc42007-05-14 18:24:23 +10001007 */
1008 if ((bp->b_error == EOPNOTSUPP) &&
1009 (bp->b_flags & (XBF_ORDERED|XBF_ASYNC)) == (XBF_ORDERED|XBF_ASYNC)) {
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001010 trace_xfs_buf_ordered_retry(bp, _RET_IP_);
David Chinner0bfefc42007-05-14 18:24:23 +10001011 bp->b_flags &= ~XBF_ORDERED;
Christoph Hellwig73f6aa42008-10-10 17:28:29 +11001012 bp->b_flags |= _XFS_BARRIER_FAILED;
David Chinner0bfefc42007-05-14 18:24:23 +10001013 xfs_buf_iorequest(bp);
1014 } else if (bp->b_iodone)
Nathan Scottce8e9222006-01-11 15:39:08 +11001015 (*(bp->b_iodone))(bp);
1016 else if (bp->b_flags & XBF_ASYNC)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001017 xfs_buf_relse(bp);
1018}
1019
1020void
Nathan Scottce8e9222006-01-11 15:39:08 +11001021xfs_buf_ioend(
1022 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001023 int schedule)
1024{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001025 trace_xfs_buf_iodone(bp, _RET_IP_);
1026
Lachlan McIlroy77be55a2007-11-23 16:31:00 +11001027 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
Nathan Scottce8e9222006-01-11 15:39:08 +11001028 if (bp->b_error == 0)
1029 bp->b_flags |= XBF_DONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001030
Nathan Scottce8e9222006-01-11 15:39:08 +11001031 if ((bp->b_iodone) || (bp->b_flags & XBF_ASYNC)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001032 if (schedule) {
David Howellsc4028952006-11-22 14:57:56 +00001033 INIT_WORK(&bp->b_iodone_work, xfs_buf_iodone_work);
Nathan Scottce8e9222006-01-11 15:39:08 +11001034 queue_work(xfslogd_workqueue, &bp->b_iodone_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001035 } else {
David Howellsc4028952006-11-22 14:57:56 +00001036 xfs_buf_iodone_work(&bp->b_iodone_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001037 }
1038 } else {
David Chinnerb4dd3302008-08-13 16:36:11 +10001039 complete(&bp->b_iowait);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001040 }
1041}
1042
Linus Torvalds1da177e2005-04-16 15:20:36 -07001043void
Nathan Scottce8e9222006-01-11 15:39:08 +11001044xfs_buf_ioerror(
1045 xfs_buf_t *bp,
1046 int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001047{
1048 ASSERT(error >= 0 && error <= 0xffff);
Nathan Scottce8e9222006-01-11 15:39:08 +11001049 bp->b_error = (unsigned short)error;
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001050 trace_xfs_buf_ioerror(bp, error, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001051}
1052
Linus Torvalds1da177e2005-04-16 15:20:36 -07001053int
Christoph Hellwig64e0bc72010-01-13 22:17:58 +00001054xfs_bwrite(
1055 struct xfs_mount *mp,
1056 struct xfs_buf *bp)
1057{
1058 int iowait = (bp->b_flags & XBF_ASYNC) == 0;
1059 int error = 0;
1060
1061 bp->b_strat = xfs_bdstrat_cb;
1062 bp->b_mount = mp;
1063 bp->b_flags |= XBF_WRITE;
1064 if (!iowait)
1065 bp->b_flags |= _XBF_RUN_QUEUES;
1066
1067 xfs_buf_delwri_dequeue(bp);
1068 xfs_buf_iostrategy(bp);
1069
1070 if (iowait) {
1071 error = xfs_buf_iowait(bp);
1072 if (error)
1073 xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
1074 xfs_buf_relse(bp);
1075 }
1076
1077 return error;
1078}
1079
1080int
Christoph Hellwig5d765b92008-12-03 12:20:26 +01001081xfs_bawrite(
1082 void *mp,
1083 struct xfs_buf *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001084{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001085 trace_xfs_buf_bawrite(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001086
Christoph Hellwig5d765b92008-12-03 12:20:26 +01001087 ASSERT(bp->b_bn != XFS_BUF_DADDR_NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001088
Christoph Hellwig5d765b92008-12-03 12:20:26 +01001089 xfs_buf_delwri_dequeue(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001090
Christoph Hellwig5d765b92008-12-03 12:20:26 +01001091 bp->b_flags &= ~(XBF_READ | XBF_DELWRI | XBF_READ_AHEAD);
1092 bp->b_flags |= (XBF_WRITE | XBF_ASYNC | _XBF_RUN_QUEUES);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001093
Christoph Hellwig15ac08a2008-12-09 04:47:30 -05001094 bp->b_mount = mp;
Christoph Hellwig5d765b92008-12-03 12:20:26 +01001095 bp->b_strat = xfs_bdstrat_cb;
1096 return xfs_bdstrat_cb(bp);
1097}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001098
Christoph Hellwig5d765b92008-12-03 12:20:26 +01001099void
1100xfs_bdwrite(
1101 void *mp,
1102 struct xfs_buf *bp)
1103{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001104 trace_xfs_buf_bdwrite(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001105
Christoph Hellwig5d765b92008-12-03 12:20:26 +01001106 bp->b_strat = xfs_bdstrat_cb;
Christoph Hellwig15ac08a2008-12-09 04:47:30 -05001107 bp->b_mount = mp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108
Christoph Hellwig5d765b92008-12-03 12:20:26 +01001109 bp->b_flags &= ~XBF_READ;
1110 bp->b_flags |= (XBF_DELWRI | XBF_ASYNC);
1111
1112 xfs_buf_delwri_queue(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001113}
1114
Christoph Hellwigb8f82a42009-11-14 16:17:22 +00001115STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001116_xfs_buf_ioend(
1117 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001118 int schedule)
1119{
Christoph Hellwig6ab455e2008-05-19 16:34:42 +10001120 if (atomic_dec_and_test(&bp->b_io_remaining) == 1) {
1121 bp->b_flags &= ~_XBF_PAGE_LOCKED;
Nathan Scottce8e9222006-01-11 15:39:08 +11001122 xfs_buf_ioend(bp, schedule);
Christoph Hellwig6ab455e2008-05-19 16:34:42 +10001123 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001124}
1125
Al Viro782e3b32007-10-12 07:17:47 +01001126STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001127xfs_buf_bio_end_io(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001128 struct bio *bio,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001129 int error)
1130{
Nathan Scottce8e9222006-01-11 15:39:08 +11001131 xfs_buf_t *bp = (xfs_buf_t *)bio->bi_private;
1132 unsigned int blocksize = bp->b_target->bt_bsize;
Nathan Scotteedb5532005-09-02 16:39:56 +10001133 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001134
Lachlan McIlroycfbe5262008-12-12 15:27:25 +11001135 xfs_buf_ioerror(bp, -error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001136
Nathan Scotteedb5532005-09-02 16:39:56 +10001137 do {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001138 struct page *page = bvec->bv_page;
1139
Nathan Scott948ecdb2006-09-28 11:03:13 +10001140 ASSERT(!PagePrivate(page));
Nathan Scottce8e9222006-01-11 15:39:08 +11001141 if (unlikely(bp->b_error)) {
1142 if (bp->b_flags & XBF_READ)
Nathan Scotteedb5532005-09-02 16:39:56 +10001143 ClearPageUptodate(page);
Nathan Scottce8e9222006-01-11 15:39:08 +11001144 } else if (blocksize >= PAGE_CACHE_SIZE) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001145 SetPageUptodate(page);
1146 } else if (!PagePrivate(page) &&
Nathan Scottce8e9222006-01-11 15:39:08 +11001147 (bp->b_flags & _XBF_PAGE_CACHE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001148 set_page_region(page, bvec->bv_offset, bvec->bv_len);
1149 }
1150
Nathan Scotteedb5532005-09-02 16:39:56 +10001151 if (--bvec >= bio->bi_io_vec)
1152 prefetchw(&bvec->bv_page->flags);
Christoph Hellwig6ab455e2008-05-19 16:34:42 +10001153
1154 if (bp->b_flags & _XBF_PAGE_LOCKED)
1155 unlock_page(page);
Nathan Scotteedb5532005-09-02 16:39:56 +10001156 } while (bvec >= bio->bi_io_vec);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001157
Nathan Scottce8e9222006-01-11 15:39:08 +11001158 _xfs_buf_ioend(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001159 bio_put(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001160}
1161
1162STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001163_xfs_buf_ioapply(
1164 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001165{
Christoph Hellwiga9759f22007-12-07 14:07:08 +11001166 int rw, map_i, total_nr_pages, nr_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001167 struct bio *bio;
Nathan Scottce8e9222006-01-11 15:39:08 +11001168 int offset = bp->b_offset;
1169 int size = bp->b_count_desired;
1170 sector_t sector = bp->b_bn;
1171 unsigned int blocksize = bp->b_target->bt_bsize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001172
Nathan Scottce8e9222006-01-11 15:39:08 +11001173 total_nr_pages = bp->b_page_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001174 map_i = 0;
1175
Nathan Scottce8e9222006-01-11 15:39:08 +11001176 if (bp->b_flags & XBF_ORDERED) {
1177 ASSERT(!(bp->b_flags & XBF_READ));
Christoph Hellwigf538d4d2005-11-02 10:26:59 +11001178 rw = WRITE_BARRIER;
Dave Chinner2ee1aba2009-11-24 18:03:15 +00001179 } else if (bp->b_flags & XBF_LOG_BUFFER) {
Nathan Scott51bdd702006-09-28 11:01:57 +10001180 ASSERT(!(bp->b_flags & XBF_READ_AHEAD));
1181 bp->b_flags &= ~_XBF_RUN_QUEUES;
1182 rw = (bp->b_flags & XBF_WRITE) ? WRITE_SYNC : READ_SYNC;
Dave Chinner2ee1aba2009-11-24 18:03:15 +00001183 } else if (bp->b_flags & _XBF_RUN_QUEUES) {
1184 ASSERT(!(bp->b_flags & XBF_READ_AHEAD));
1185 bp->b_flags &= ~_XBF_RUN_QUEUES;
1186 rw = (bp->b_flags & XBF_WRITE) ? WRITE_META : READ_META;
Nathan Scott51bdd702006-09-28 11:01:57 +10001187 } else {
1188 rw = (bp->b_flags & XBF_WRITE) ? WRITE :
1189 (bp->b_flags & XBF_READ_AHEAD) ? READA : READ;
Christoph Hellwigf538d4d2005-11-02 10:26:59 +11001190 }
1191
Nathan Scottce8e9222006-01-11 15:39:08 +11001192 /* Special code path for reading a sub page size buffer in --
Linus Torvalds1da177e2005-04-16 15:20:36 -07001193 * we populate up the whole page, and hence the other metadata
1194 * in the same page. This optimization is only valid when the
Nathan Scottce8e9222006-01-11 15:39:08 +11001195 * filesystem block size is not smaller than the page size.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001196 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001197 if ((bp->b_buffer_length < PAGE_CACHE_SIZE) &&
Christoph Hellwig6ab455e2008-05-19 16:34:42 +10001198 ((bp->b_flags & (XBF_READ|_XBF_PAGE_LOCKED)) ==
1199 (XBF_READ|_XBF_PAGE_LOCKED)) &&
Nathan Scottce8e9222006-01-11 15:39:08 +11001200 (blocksize >= PAGE_CACHE_SIZE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001201 bio = bio_alloc(GFP_NOIO, 1);
1202
Nathan Scottce8e9222006-01-11 15:39:08 +11001203 bio->bi_bdev = bp->b_target->bt_bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001204 bio->bi_sector = sector - (offset >> BBSHIFT);
Nathan Scottce8e9222006-01-11 15:39:08 +11001205 bio->bi_end_io = xfs_buf_bio_end_io;
1206 bio->bi_private = bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001207
Nathan Scottce8e9222006-01-11 15:39:08 +11001208 bio_add_page(bio, bp->b_pages[0], PAGE_CACHE_SIZE, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001209 size = 0;
1210
Nathan Scottce8e9222006-01-11 15:39:08 +11001211 atomic_inc(&bp->b_io_remaining);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001212
1213 goto submit_io;
1214 }
1215
Linus Torvalds1da177e2005-04-16 15:20:36 -07001216next_chunk:
Nathan Scottce8e9222006-01-11 15:39:08 +11001217 atomic_inc(&bp->b_io_remaining);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001218 nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1219 if (nr_pages > total_nr_pages)
1220 nr_pages = total_nr_pages;
1221
1222 bio = bio_alloc(GFP_NOIO, nr_pages);
Nathan Scottce8e9222006-01-11 15:39:08 +11001223 bio->bi_bdev = bp->b_target->bt_bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001224 bio->bi_sector = sector;
Nathan Scottce8e9222006-01-11 15:39:08 +11001225 bio->bi_end_io = xfs_buf_bio_end_io;
1226 bio->bi_private = bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001227
1228 for (; size && nr_pages; nr_pages--, map_i++) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001229 int rbytes, nbytes = PAGE_CACHE_SIZE - offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001230
1231 if (nbytes > size)
1232 nbytes = size;
1233
Nathan Scottce8e9222006-01-11 15:39:08 +11001234 rbytes = bio_add_page(bio, bp->b_pages[map_i], nbytes, offset);
1235 if (rbytes < nbytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001236 break;
1237
1238 offset = 0;
1239 sector += nbytes >> BBSHIFT;
1240 size -= nbytes;
1241 total_nr_pages--;
1242 }
1243
1244submit_io:
1245 if (likely(bio->bi_size)) {
1246 submit_bio(rw, bio);
1247 if (size)
1248 goto next_chunk;
1249 } else {
1250 bio_put(bio);
Nathan Scottce8e9222006-01-11 15:39:08 +11001251 xfs_buf_ioerror(bp, EIO);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001252 }
1253}
1254
Linus Torvalds1da177e2005-04-16 15:20:36 -07001255int
Nathan Scottce8e9222006-01-11 15:39:08 +11001256xfs_buf_iorequest(
1257 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001258{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001259 trace_xfs_buf_iorequest(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001260
Nathan Scottce8e9222006-01-11 15:39:08 +11001261 if (bp->b_flags & XBF_DELWRI) {
1262 xfs_buf_delwri_queue(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001263 return 0;
1264 }
1265
Nathan Scottce8e9222006-01-11 15:39:08 +11001266 if (bp->b_flags & XBF_WRITE) {
1267 xfs_buf_wait_unpin(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001268 }
1269
Nathan Scottce8e9222006-01-11 15:39:08 +11001270 xfs_buf_hold(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001271
1272 /* Set the count to 1 initially, this will stop an I/O
1273 * completion callout which happens before we have started
Nathan Scottce8e9222006-01-11 15:39:08 +11001274 * all the I/O from calling xfs_buf_ioend too early.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001275 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001276 atomic_set(&bp->b_io_remaining, 1);
1277 _xfs_buf_ioapply(bp);
1278 _xfs_buf_ioend(bp, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001279
Nathan Scottce8e9222006-01-11 15:39:08 +11001280 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001281 return 0;
1282}
1283
1284/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001285 * Waits for I/O to complete on the buffer supplied.
1286 * It returns immediately if no I/O is pending.
1287 * It returns the I/O error code, if any, or 0 if there was no error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001288 */
1289int
Nathan Scottce8e9222006-01-11 15:39:08 +11001290xfs_buf_iowait(
1291 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001292{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001293 trace_xfs_buf_iowait(bp, _RET_IP_);
1294
Nathan Scottce8e9222006-01-11 15:39:08 +11001295 if (atomic_read(&bp->b_io_remaining))
1296 blk_run_address_space(bp->b_target->bt_mapping);
David Chinnerb4dd3302008-08-13 16:36:11 +10001297 wait_for_completion(&bp->b_iowait);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001298
1299 trace_xfs_buf_iowait_done(bp, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +11001300 return bp->b_error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001301}
1302
Nathan Scottce8e9222006-01-11 15:39:08 +11001303xfs_caddr_t
1304xfs_buf_offset(
1305 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001306 size_t offset)
1307{
1308 struct page *page;
1309
Nathan Scottce8e9222006-01-11 15:39:08 +11001310 if (bp->b_flags & XBF_MAPPED)
1311 return XFS_BUF_PTR(bp) + offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001312
Nathan Scottce8e9222006-01-11 15:39:08 +11001313 offset += bp->b_offset;
1314 page = bp->b_pages[offset >> PAGE_CACHE_SHIFT];
1315 return (xfs_caddr_t)page_address(page) + (offset & (PAGE_CACHE_SIZE-1));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001316}
1317
1318/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001319 * Move data into or out of a buffer.
1320 */
1321void
Nathan Scottce8e9222006-01-11 15:39:08 +11001322xfs_buf_iomove(
1323 xfs_buf_t *bp, /* buffer to process */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001324 size_t boff, /* starting buffer offset */
1325 size_t bsize, /* length to copy */
1326 caddr_t data, /* data address */
Nathan Scottce8e9222006-01-11 15:39:08 +11001327 xfs_buf_rw_t mode) /* read/write/zero flag */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001328{
1329 size_t bend, cpoff, csize;
1330 struct page *page;
1331
1332 bend = boff + bsize;
1333 while (boff < bend) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001334 page = bp->b_pages[xfs_buf_btoct(boff + bp->b_offset)];
1335 cpoff = xfs_buf_poff(boff + bp->b_offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001336 csize = min_t(size_t,
Nathan Scottce8e9222006-01-11 15:39:08 +11001337 PAGE_CACHE_SIZE-cpoff, bp->b_count_desired-boff);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001338
1339 ASSERT(((csize + cpoff) <= PAGE_CACHE_SIZE));
1340
1341 switch (mode) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001342 case XBRW_ZERO:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001343 memset(page_address(page) + cpoff, 0, csize);
1344 break;
Nathan Scottce8e9222006-01-11 15:39:08 +11001345 case XBRW_READ:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001346 memcpy(data, page_address(page) + cpoff, csize);
1347 break;
Nathan Scottce8e9222006-01-11 15:39:08 +11001348 case XBRW_WRITE:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001349 memcpy(page_address(page) + cpoff, data, csize);
1350 }
1351
1352 boff += csize;
1353 data += csize;
1354 }
1355}
1356
1357/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001358 * Handling of buffer targets (buftargs).
Linus Torvalds1da177e2005-04-16 15:20:36 -07001359 */
1360
1361/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001362 * Wait for any bufs with callbacks that have been submitted but
1363 * have not yet returned... walk the hash list for the target.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001364 */
1365void
1366xfs_wait_buftarg(
1367 xfs_buftarg_t *btp)
1368{
1369 xfs_buf_t *bp, *n;
1370 xfs_bufhash_t *hash;
1371 uint i;
1372
1373 for (i = 0; i < (1 << btp->bt_hashshift); i++) {
1374 hash = &btp->bt_hash[i];
1375again:
1376 spin_lock(&hash->bh_lock);
Nathan Scottce8e9222006-01-11 15:39:08 +11001377 list_for_each_entry_safe(bp, n, &hash->bh_list, b_hash_list) {
1378 ASSERT(btp == bp->b_target);
1379 if (!(bp->b_flags & XBF_FS_MANAGED)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001380 spin_unlock(&hash->bh_lock);
David Chinner2f926582005-09-05 08:33:35 +10001381 /*
1382 * Catch superblock reference count leaks
1383 * immediately
1384 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001385 BUG_ON(bp->b_bn == 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001386 delay(100);
1387 goto again;
1388 }
1389 }
1390 spin_unlock(&hash->bh_lock);
1391 }
1392}
1393
1394/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001395 * Allocate buffer hash table for a given target.
1396 * For devices containing metadata (i.e. not the log/realtime devices)
1397 * we need to allocate a much larger hash table.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001398 */
1399STATIC void
1400xfs_alloc_bufhash(
1401 xfs_buftarg_t *btp,
1402 int external)
1403{
1404 unsigned int i;
1405
1406 btp->bt_hashshift = external ? 3 : 8; /* 8 or 256 buckets */
1407 btp->bt_hashmask = (1 << btp->bt_hashshift) - 1;
1408 btp->bt_hash = kmem_zalloc((1 << btp->bt_hashshift) *
Vlad Apostolov93c189c2006-11-11 18:03:49 +11001409 sizeof(xfs_bufhash_t), KM_SLEEP | KM_LARGE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001410 for (i = 0; i < (1 << btp->bt_hashshift); i++) {
1411 spin_lock_init(&btp->bt_hash[i].bh_lock);
1412 INIT_LIST_HEAD(&btp->bt_hash[i].bh_list);
1413 }
1414}
1415
1416STATIC void
1417xfs_free_bufhash(
1418 xfs_buftarg_t *btp)
1419{
Denys Vlasenkof0e2d932008-05-19 16:31:57 +10001420 kmem_free(btp->bt_hash);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001421 btp->bt_hash = NULL;
1422}
1423
David Chinnera6867a62006-01-11 15:37:58 +11001424/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001425 * buftarg list for delwrite queue processing
David Chinnera6867a62006-01-11 15:37:58 +11001426 */
Tim Shimmine6a0e9c2007-05-08 13:49:59 +10001427static LIST_HEAD(xfs_buftarg_list);
David Chinner7989cb82007-02-10 18:34:56 +11001428static DEFINE_SPINLOCK(xfs_buftarg_lock);
David Chinnera6867a62006-01-11 15:37:58 +11001429
1430STATIC void
1431xfs_register_buftarg(
1432 xfs_buftarg_t *btp)
1433{
1434 spin_lock(&xfs_buftarg_lock);
1435 list_add(&btp->bt_list, &xfs_buftarg_list);
1436 spin_unlock(&xfs_buftarg_lock);
1437}
1438
1439STATIC void
1440xfs_unregister_buftarg(
1441 xfs_buftarg_t *btp)
1442{
1443 spin_lock(&xfs_buftarg_lock);
1444 list_del(&btp->bt_list);
1445 spin_unlock(&xfs_buftarg_lock);
1446}
1447
Linus Torvalds1da177e2005-04-16 15:20:36 -07001448void
1449xfs_free_buftarg(
Christoph Hellwigb7963132009-03-03 14:48:37 -05001450 struct xfs_mount *mp,
1451 struct xfs_buftarg *btp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001452{
1453 xfs_flush_buftarg(btp, 1);
Christoph Hellwigb7963132009-03-03 14:48:37 -05001454 if (mp->m_flags & XFS_MOUNT_BARRIER)
1455 xfs_blkdev_issue_flush(btp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001456 xfs_free_bufhash(btp);
Nathan Scottce8e9222006-01-11 15:39:08 +11001457 iput(btp->bt_mapping->host);
David Chinnera6867a62006-01-11 15:37:58 +11001458
Nathan Scottce8e9222006-01-11 15:39:08 +11001459 /* Unregister the buftarg first so that we don't get a
1460 * wakeup finding a non-existent task
1461 */
David Chinnera6867a62006-01-11 15:37:58 +11001462 xfs_unregister_buftarg(btp);
1463 kthread_stop(btp->bt_task);
1464
Denys Vlasenkof0e2d932008-05-19 16:31:57 +10001465 kmem_free(btp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001466}
1467
Linus Torvalds1da177e2005-04-16 15:20:36 -07001468STATIC int
1469xfs_setsize_buftarg_flags(
1470 xfs_buftarg_t *btp,
1471 unsigned int blocksize,
1472 unsigned int sectorsize,
1473 int verbose)
1474{
Nathan Scottce8e9222006-01-11 15:39:08 +11001475 btp->bt_bsize = blocksize;
1476 btp->bt_sshift = ffs(sectorsize) - 1;
1477 btp->bt_smask = sectorsize - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001478
Nathan Scottce8e9222006-01-11 15:39:08 +11001479 if (set_blocksize(btp->bt_bdev, sectorsize)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001480 printk(KERN_WARNING
1481 "XFS: Cannot set_blocksize to %u on device %s\n",
1482 sectorsize, XFS_BUFTARG_NAME(btp));
1483 return EINVAL;
1484 }
1485
1486 if (verbose &&
1487 (PAGE_CACHE_SIZE / BITS_PER_LONG) > sectorsize) {
1488 printk(KERN_WARNING
1489 "XFS: %u byte sectors in use on device %s. "
1490 "This is suboptimal; %u or greater is ideal.\n",
1491 sectorsize, XFS_BUFTARG_NAME(btp),
1492 (unsigned int)PAGE_CACHE_SIZE / BITS_PER_LONG);
1493 }
1494
1495 return 0;
1496}
1497
1498/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001499 * When allocating the initial buffer target we have not yet
1500 * read in the superblock, so don't know what sized sectors
1501 * are being used is at this early stage. Play safe.
1502 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001503STATIC int
1504xfs_setsize_buftarg_early(
1505 xfs_buftarg_t *btp,
1506 struct block_device *bdev)
1507{
1508 return xfs_setsize_buftarg_flags(btp,
Martin K. Petersene1defc42009-05-22 17:17:49 -04001509 PAGE_CACHE_SIZE, bdev_logical_block_size(bdev), 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001510}
1511
1512int
1513xfs_setsize_buftarg(
1514 xfs_buftarg_t *btp,
1515 unsigned int blocksize,
1516 unsigned int sectorsize)
1517{
1518 return xfs_setsize_buftarg_flags(btp, blocksize, sectorsize, 1);
1519}
1520
1521STATIC int
1522xfs_mapping_buftarg(
1523 xfs_buftarg_t *btp,
1524 struct block_device *bdev)
1525{
1526 struct backing_dev_info *bdi;
1527 struct inode *inode;
1528 struct address_space *mapping;
Christoph Hellwigf5e54d62006-06-28 04:26:44 -07001529 static const struct address_space_operations mapping_aops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001530 .sync_page = block_sync_page,
Christoph Lametere965f962006-02-01 03:05:41 -08001531 .migratepage = fail_migrate_page,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001532 };
1533
1534 inode = new_inode(bdev->bd_inode->i_sb);
1535 if (!inode) {
1536 printk(KERN_WARNING
1537 "XFS: Cannot allocate mapping inode for device %s\n",
1538 XFS_BUFTARG_NAME(btp));
1539 return ENOMEM;
1540 }
1541 inode->i_mode = S_IFBLK;
1542 inode->i_bdev = bdev;
1543 inode->i_rdev = bdev->bd_dev;
1544 bdi = blk_get_backing_dev_info(bdev);
1545 if (!bdi)
1546 bdi = &default_backing_dev_info;
1547 mapping = &inode->i_data;
1548 mapping->a_ops = &mapping_aops;
1549 mapping->backing_dev_info = bdi;
1550 mapping_set_gfp_mask(mapping, GFP_NOFS);
Nathan Scottce8e9222006-01-11 15:39:08 +11001551 btp->bt_mapping = mapping;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001552 return 0;
1553}
1554
David Chinnera6867a62006-01-11 15:37:58 +11001555STATIC int
1556xfs_alloc_delwrite_queue(
1557 xfs_buftarg_t *btp)
1558{
1559 int error = 0;
1560
1561 INIT_LIST_HEAD(&btp->bt_list);
1562 INIT_LIST_HEAD(&btp->bt_delwrite_queue);
Eric Sandeen007c61c2007-10-11 17:43:56 +10001563 spin_lock_init(&btp->bt_delwrite_lock);
David Chinnera6867a62006-01-11 15:37:58 +11001564 btp->bt_flags = 0;
1565 btp->bt_task = kthread_run(xfsbufd, btp, "xfsbufd");
1566 if (IS_ERR(btp->bt_task)) {
1567 error = PTR_ERR(btp->bt_task);
1568 goto out_error;
1569 }
1570 xfs_register_buftarg(btp);
1571out_error:
1572 return error;
1573}
1574
Linus Torvalds1da177e2005-04-16 15:20:36 -07001575xfs_buftarg_t *
1576xfs_alloc_buftarg(
1577 struct block_device *bdev,
1578 int external)
1579{
1580 xfs_buftarg_t *btp;
1581
1582 btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
1583
Nathan Scottce8e9222006-01-11 15:39:08 +11001584 btp->bt_dev = bdev->bd_dev;
1585 btp->bt_bdev = bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001586 if (xfs_setsize_buftarg_early(btp, bdev))
1587 goto error;
1588 if (xfs_mapping_buftarg(btp, bdev))
1589 goto error;
David Chinnera6867a62006-01-11 15:37:58 +11001590 if (xfs_alloc_delwrite_queue(btp))
1591 goto error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001592 xfs_alloc_bufhash(btp, external);
1593 return btp;
1594
1595error:
Denys Vlasenkof0e2d932008-05-19 16:31:57 +10001596 kmem_free(btp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001597 return NULL;
1598}
1599
1600
1601/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001602 * Delayed write buffer handling
Linus Torvalds1da177e2005-04-16 15:20:36 -07001603 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001604STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001605xfs_buf_delwri_queue(
1606 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001607 int unlock)
1608{
Nathan Scottce8e9222006-01-11 15:39:08 +11001609 struct list_head *dwq = &bp->b_target->bt_delwrite_queue;
1610 spinlock_t *dwlk = &bp->b_target->bt_delwrite_lock;
David Chinnera6867a62006-01-11 15:37:58 +11001611
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001612 trace_xfs_buf_delwri_queue(bp, _RET_IP_);
1613
Nathan Scottce8e9222006-01-11 15:39:08 +11001614 ASSERT((bp->b_flags&(XBF_DELWRI|XBF_ASYNC)) == (XBF_DELWRI|XBF_ASYNC));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001615
David Chinnera6867a62006-01-11 15:37:58 +11001616 spin_lock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001617 /* If already in the queue, dequeue and place at tail */
Nathan Scottce8e9222006-01-11 15:39:08 +11001618 if (!list_empty(&bp->b_list)) {
1619 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1620 if (unlock)
1621 atomic_dec(&bp->b_hold);
1622 list_del(&bp->b_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001623 }
1624
Dave Chinnerc9c12972010-01-11 11:49:59 +00001625 if (list_empty(dwq)) {
1626 /* start xfsbufd as it is about to have something to do */
1627 wake_up_process(bp->b_target->bt_task);
1628 }
1629
Nathan Scottce8e9222006-01-11 15:39:08 +11001630 bp->b_flags |= _XBF_DELWRI_Q;
1631 list_add_tail(&bp->b_list, dwq);
1632 bp->b_queuetime = jiffies;
David Chinnera6867a62006-01-11 15:37:58 +11001633 spin_unlock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001634
1635 if (unlock)
Nathan Scottce8e9222006-01-11 15:39:08 +11001636 xfs_buf_unlock(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001637}
1638
1639void
Nathan Scottce8e9222006-01-11 15:39:08 +11001640xfs_buf_delwri_dequeue(
1641 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001642{
Nathan Scottce8e9222006-01-11 15:39:08 +11001643 spinlock_t *dwlk = &bp->b_target->bt_delwrite_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001644 int dequeued = 0;
1645
David Chinnera6867a62006-01-11 15:37:58 +11001646 spin_lock(dwlk);
Nathan Scottce8e9222006-01-11 15:39:08 +11001647 if ((bp->b_flags & XBF_DELWRI) && !list_empty(&bp->b_list)) {
1648 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1649 list_del_init(&bp->b_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001650 dequeued = 1;
1651 }
Nathan Scottce8e9222006-01-11 15:39:08 +11001652 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
David Chinnera6867a62006-01-11 15:37:58 +11001653 spin_unlock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001654
1655 if (dequeued)
Nathan Scottce8e9222006-01-11 15:39:08 +11001656 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001657
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001658 trace_xfs_buf_delwri_dequeue(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001659}
1660
1661STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001662xfs_buf_runall_queues(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001663 struct workqueue_struct *queue)
1664{
1665 flush_workqueue(queue);
1666}
1667
Linus Torvalds1da177e2005-04-16 15:20:36 -07001668STATIC int
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001669xfsbufd_wakeup(
Nathan Scott15c84a42005-11-04 10:51:01 +11001670 int priority,
1671 gfp_t mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001672{
Christoph Hellwigda7f93e2006-01-11 20:49:57 +11001673 xfs_buftarg_t *btp;
David Chinnera6867a62006-01-11 15:37:58 +11001674
1675 spin_lock(&xfs_buftarg_lock);
Christoph Hellwigda7f93e2006-01-11 20:49:57 +11001676 list_for_each_entry(btp, &xfs_buftarg_list, bt_list) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001677 if (test_bit(XBT_FORCE_SLEEP, &btp->bt_flags))
David Chinnera6867a62006-01-11 15:37:58 +11001678 continue;
Dave Chinnerc9c12972010-01-11 11:49:59 +00001679 if (list_empty(&btp->bt_delwrite_queue))
1680 continue;
Nathan Scottce8e9222006-01-11 15:39:08 +11001681 set_bit(XBT_FORCE_FLUSH, &btp->bt_flags);
David Chinnera6867a62006-01-11 15:37:58 +11001682 wake_up_process(btp->bt_task);
1683 }
1684 spin_unlock(&xfs_buftarg_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001685 return 0;
1686}
1687
David Chinner585e6d82007-02-10 18:32:29 +11001688/*
1689 * Move as many buffers as specified to the supplied list
1690 * idicating if we skipped any buffers to prevent deadlocks.
1691 */
1692STATIC int
1693xfs_buf_delwri_split(
1694 xfs_buftarg_t *target,
1695 struct list_head *list,
David Chinner5e6a07d2007-02-10 18:34:49 +11001696 unsigned long age)
David Chinner585e6d82007-02-10 18:32:29 +11001697{
1698 xfs_buf_t *bp, *n;
1699 struct list_head *dwq = &target->bt_delwrite_queue;
1700 spinlock_t *dwlk = &target->bt_delwrite_lock;
1701 int skipped = 0;
David Chinner5e6a07d2007-02-10 18:34:49 +11001702 int force;
David Chinner585e6d82007-02-10 18:32:29 +11001703
David Chinner5e6a07d2007-02-10 18:34:49 +11001704 force = test_and_clear_bit(XBT_FORCE_FLUSH, &target->bt_flags);
David Chinner585e6d82007-02-10 18:32:29 +11001705 INIT_LIST_HEAD(list);
1706 spin_lock(dwlk);
1707 list_for_each_entry_safe(bp, n, dwq, b_list) {
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001708 trace_xfs_buf_delwri_split(bp, _RET_IP_);
David Chinner585e6d82007-02-10 18:32:29 +11001709 ASSERT(bp->b_flags & XBF_DELWRI);
1710
1711 if (!xfs_buf_ispin(bp) && !xfs_buf_cond_lock(bp)) {
David Chinner5e6a07d2007-02-10 18:34:49 +11001712 if (!force &&
David Chinner585e6d82007-02-10 18:32:29 +11001713 time_before(jiffies, bp->b_queuetime + age)) {
1714 xfs_buf_unlock(bp);
1715 break;
1716 }
1717
1718 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q|
1719 _XBF_RUN_QUEUES);
1720 bp->b_flags |= XBF_WRITE;
1721 list_move_tail(&bp->b_list, list);
1722 } else
1723 skipped++;
1724 }
1725 spin_unlock(dwlk);
1726
1727 return skipped;
1728
1729}
1730
Linus Torvalds1da177e2005-04-16 15:20:36 -07001731STATIC int
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001732xfsbufd(
David Chinner585e6d82007-02-10 18:32:29 +11001733 void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001734{
David Chinner585e6d82007-02-10 18:32:29 +11001735 struct list_head tmp;
1736 xfs_buftarg_t *target = (xfs_buftarg_t *)data;
1737 int count;
1738 xfs_buf_t *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001739
Linus Torvalds1da177e2005-04-16 15:20:36 -07001740 current->flags |= PF_MEMALLOC;
1741
Rafael J. Wysocki978c7b22007-12-07 14:09:02 +11001742 set_freezable();
1743
Linus Torvalds1da177e2005-04-16 15:20:36 -07001744 do {
Dave Chinnerc9c12972010-01-11 11:49:59 +00001745 long age = xfs_buf_age_centisecs * msecs_to_jiffies(10);
1746 long tout = xfs_buf_timer_centisecs * msecs_to_jiffies(10);
1747
Christoph Lameter3e1d1d22005-06-24 23:13:50 -07001748 if (unlikely(freezing(current))) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001749 set_bit(XBT_FORCE_SLEEP, &target->bt_flags);
Christoph Lameter3e1d1d22005-06-24 23:13:50 -07001750 refrigerator();
Nathan Scottabd0cf72005-05-05 13:30:13 -07001751 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +11001752 clear_bit(XBT_FORCE_SLEEP, &target->bt_flags);
Nathan Scottabd0cf72005-05-05 13:30:13 -07001753 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001754
Dave Chinnerc9c12972010-01-11 11:49:59 +00001755 /* sleep for a long time if there is nothing to do. */
1756 if (list_empty(&target->bt_delwrite_queue))
1757 tout = MAX_SCHEDULE_TIMEOUT;
1758 schedule_timeout_interruptible(tout);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001759
Dave Chinnerc9c12972010-01-11 11:49:59 +00001760 xfs_buf_delwri_split(target, &tmp, age);
Nathan Scottf07c2252006-09-28 10:52:15 +10001761 count = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001762 while (!list_empty(&tmp)) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001763 bp = list_entry(tmp.next, xfs_buf_t, b_list);
1764 ASSERT(target == bp->b_target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001765
Nathan Scottce8e9222006-01-11 15:39:08 +11001766 list_del_init(&bp->b_list);
1767 xfs_buf_iostrategy(bp);
David Chinner585e6d82007-02-10 18:32:29 +11001768 count++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001769 }
1770
Felix Blyakher3a011a12009-02-18 15:56:51 -06001771 if (as_list_len > 0)
1772 purge_addresses();
Nathan Scottf07c2252006-09-28 10:52:15 +10001773 if (count)
1774 blk_run_address_space(target->bt_mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001775
Christoph Hellwig4df08c52005-09-05 08:34:18 +10001776 } while (!kthread_should_stop());
Linus Torvalds1da177e2005-04-16 15:20:36 -07001777
Christoph Hellwig4df08c52005-09-05 08:34:18 +10001778 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001779}
1780
1781/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001782 * Go through all incore buffers, and release buffers if they belong to
1783 * the given device. This is used in filesystem error handling to
1784 * preserve the consistency of its metadata.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001785 */
1786int
1787xfs_flush_buftarg(
David Chinner585e6d82007-02-10 18:32:29 +11001788 xfs_buftarg_t *target,
1789 int wait)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001790{
David Chinner585e6d82007-02-10 18:32:29 +11001791 struct list_head tmp;
1792 xfs_buf_t *bp, *n;
1793 int pincount = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001794
Dave Chinnerc626d172009-04-06 18:42:11 +02001795 xfs_buf_runall_queues(xfsconvertd_workqueue);
Nathan Scottce8e9222006-01-11 15:39:08 +11001796 xfs_buf_runall_queues(xfsdatad_workqueue);
1797 xfs_buf_runall_queues(xfslogd_workqueue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001798
David Chinner5e6a07d2007-02-10 18:34:49 +11001799 set_bit(XBT_FORCE_FLUSH, &target->bt_flags);
1800 pincount = xfs_buf_delwri_split(target, &tmp, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001801
1802 /*
1803 * Dropped the delayed write list lock, now walk the temporary list
1804 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001805 list_for_each_entry_safe(bp, n, &tmp, b_list) {
David Chinner585e6d82007-02-10 18:32:29 +11001806 ASSERT(target == bp->b_target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001807 if (wait)
Nathan Scottce8e9222006-01-11 15:39:08 +11001808 bp->b_flags &= ~XBF_ASYNC;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001809 else
Nathan Scottce8e9222006-01-11 15:39:08 +11001810 list_del_init(&bp->b_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001811
Nathan Scottce8e9222006-01-11 15:39:08 +11001812 xfs_buf_iostrategy(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001813 }
1814
Nathan Scottf07c2252006-09-28 10:52:15 +10001815 if (wait)
1816 blk_run_address_space(target->bt_mapping);
1817
Linus Torvalds1da177e2005-04-16 15:20:36 -07001818 /*
1819 * Remaining list items must be flushed before returning
1820 */
1821 while (!list_empty(&tmp)) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001822 bp = list_entry(tmp.next, xfs_buf_t, b_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001823
Nathan Scottce8e9222006-01-11 15:39:08 +11001824 list_del_init(&bp->b_list);
1825 xfs_iowait(bp);
1826 xfs_buf_relse(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001827 }
1828
Linus Torvalds1da177e2005-04-16 15:20:36 -07001829 return pincount;
1830}
1831
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001832int __init
Nathan Scottce8e9222006-01-11 15:39:08 +11001833xfs_buf_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001834{
Nathan Scott87582802006-03-14 13:18:19 +11001835 xfs_buf_zone = kmem_zone_init_flags(sizeof(xfs_buf_t), "xfs_buf",
1836 KM_ZONE_HWALIGN, NULL);
Nathan Scottce8e9222006-01-11 15:39:08 +11001837 if (!xfs_buf_zone)
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001838 goto out;
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001839
Rafael J. Wysockib4337692007-03-22 00:11:27 -08001840 xfslogd_workqueue = create_workqueue("xfslogd");
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001841 if (!xfslogd_workqueue)
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001842 goto out_free_buf_zone;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001843
Rafael J. Wysockib4337692007-03-22 00:11:27 -08001844 xfsdatad_workqueue = create_workqueue("xfsdatad");
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001845 if (!xfsdatad_workqueue)
1846 goto out_destroy_xfslogd_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001847
Dave Chinnerc626d172009-04-06 18:42:11 +02001848 xfsconvertd_workqueue = create_workqueue("xfsconvertd");
1849 if (!xfsconvertd_workqueue)
1850 goto out_destroy_xfsdatad_workqueue;
1851
Rusty Russell8e1f9362007-07-17 04:03:17 -07001852 register_shrinker(&xfs_buf_shake);
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001853 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001854
Dave Chinnerc626d172009-04-06 18:42:11 +02001855 out_destroy_xfsdatad_workqueue:
1856 destroy_workqueue(xfsdatad_workqueue);
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001857 out_destroy_xfslogd_workqueue:
1858 destroy_workqueue(xfslogd_workqueue);
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001859 out_free_buf_zone:
Nathan Scottce8e9222006-01-11 15:39:08 +11001860 kmem_zone_destroy(xfs_buf_zone);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001861 out:
Nathan Scott87582802006-03-14 13:18:19 +11001862 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001863}
1864
Linus Torvalds1da177e2005-04-16 15:20:36 -07001865void
Nathan Scottce8e9222006-01-11 15:39:08 +11001866xfs_buf_terminate(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001867{
Rusty Russell8e1f9362007-07-17 04:03:17 -07001868 unregister_shrinker(&xfs_buf_shake);
Dave Chinnerc626d172009-04-06 18:42:11 +02001869 destroy_workqueue(xfsconvertd_workqueue);
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001870 destroy_workqueue(xfsdatad_workqueue);
1871 destroy_workqueue(xfslogd_workqueue);
Nathan Scottce8e9222006-01-11 15:39:08 +11001872 kmem_zone_destroy(xfs_buf_zone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001873}
Tim Shimmine6a0e9c2007-05-08 13:49:59 +10001874
1875#ifdef CONFIG_KDB_MODULES
1876struct list_head *
1877xfs_get_buftarg_list(void)
1878{
1879 return &xfs_buftarg_list;
1880}
1881#endif