blob: aa1d353def29c64903cf02db9af159d612259b87 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Nathan Scottf07c2252006-09-28 10:52:15 +10002 * Copyright (c) 2000-2006 Silicon Graphics, Inc.
Nathan Scott7b718762005-11-02 14:58:39 +11003 * All Rights Reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
Nathan Scott7b718762005-11-02 14:58:39 +11005 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License as
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 * published by the Free Software Foundation.
8 *
Nathan Scott7b718762005-11-02 14:58:39 +11009 * This program is distributed in the hope that it would be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 *
Nathan Scott7b718762005-11-02 14:58:39 +110014 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
Linus Torvalds1da177e2005-04-16 15:20:36 -070017 */
Vlad Apostolov93c189c2006-11-11 18:03:49 +110018#include "xfs.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/stddef.h>
20#include <linux/errno.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090021#include <linux/gfp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022#include <linux/pagemap.h>
23#include <linux/init.h>
24#include <linux/vmalloc.h>
25#include <linux/bio.h>
26#include <linux/sysctl.h>
27#include <linux/proc_fs.h>
28#include <linux/workqueue.h>
29#include <linux/percpu.h>
30#include <linux/blkdev.h>
31#include <linux/hash.h>
Christoph Hellwig4df08c52005-09-05 08:34:18 +100032#include <linux/kthread.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080033#include <linux/migrate.h>
Andrew Morton3fcfab12006-10-19 23:28:16 -070034#include <linux/backing-dev.h>
Nigel Cunningham7dfb7102006-12-06 20:34:23 -080035#include <linux/freezer.h>
Dave Chinner089716a2010-01-26 15:13:25 +110036#include <linux/list_sort.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070037
Christoph Hellwigb7963132009-03-03 14:48:37 -050038#include "xfs_sb.h"
39#include "xfs_inum.h"
Dave Chinnered3b4d62010-05-21 12:07:08 +100040#include "xfs_log.h"
Christoph Hellwigb7963132009-03-03 14:48:37 -050041#include "xfs_ag.h"
Christoph Hellwigb7963132009-03-03 14:48:37 -050042#include "xfs_mount.h"
Christoph Hellwig0b1b2132009-12-14 23:14:59 +000043#include "xfs_trace.h"
Christoph Hellwigb7963132009-03-03 14:48:37 -050044
David Chinner7989cb82007-02-10 18:34:56 +110045static kmem_zone_t *xfs_buf_zone;
David Chinnera6867a62006-01-11 15:37:58 +110046STATIC int xfsbufd(void *);
Dave Chinner7f8275d2010-07-19 14:56:17 +100047STATIC int xfsbufd_wakeup(struct shrinker *, int, gfp_t);
Nathan Scottce8e9222006-01-11 15:39:08 +110048STATIC void xfs_buf_delwri_queue(xfs_buf_t *, int);
Rusty Russell8e1f9362007-07-17 04:03:17 -070049static struct shrinker xfs_buf_shake = {
50 .shrink = xfsbufd_wakeup,
51 .seeks = DEFAULT_SEEKS,
52};
Christoph Hellwig23ea4032005-06-21 15:14:01 +100053
David Chinner7989cb82007-02-10 18:34:56 +110054static struct workqueue_struct *xfslogd_workqueue;
Christoph Hellwig0829c362005-09-02 16:58:49 +100055struct workqueue_struct *xfsdatad_workqueue;
Dave Chinnerc626d172009-04-06 18:42:11 +020056struct workqueue_struct *xfsconvertd_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070057
Nathan Scottce8e9222006-01-11 15:39:08 +110058#ifdef XFS_BUF_LOCK_TRACKING
59# define XB_SET_OWNER(bp) ((bp)->b_last_holder = current->pid)
60# define XB_CLEAR_OWNER(bp) ((bp)->b_last_holder = -1)
61# define XB_GET_OWNER(bp) ((bp)->b_last_holder)
Linus Torvalds1da177e2005-04-16 15:20:36 -070062#else
Nathan Scottce8e9222006-01-11 15:39:08 +110063# define XB_SET_OWNER(bp) do { } while (0)
64# define XB_CLEAR_OWNER(bp) do { } while (0)
65# define XB_GET_OWNER(bp) do { } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -070066#endif
67
Nathan Scottce8e9222006-01-11 15:39:08 +110068#define xb_to_gfp(flags) \
69 ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : \
70 ((flags) & XBF_DONT_BLOCK) ? GFP_NOFS : GFP_KERNEL) | __GFP_NOWARN)
Linus Torvalds1da177e2005-04-16 15:20:36 -070071
Nathan Scottce8e9222006-01-11 15:39:08 +110072#define xb_to_km(flags) \
73 (((flags) & XBF_DONT_BLOCK) ? KM_NOFS : KM_SLEEP)
Linus Torvalds1da177e2005-04-16 15:20:36 -070074
Nathan Scottce8e9222006-01-11 15:39:08 +110075#define xfs_buf_allocate(flags) \
76 kmem_zone_alloc(xfs_buf_zone, xb_to_km(flags))
77#define xfs_buf_deallocate(bp) \
78 kmem_zone_free(xfs_buf_zone, (bp));
Linus Torvalds1da177e2005-04-16 15:20:36 -070079
James Bottomley73c77e22010-01-25 11:42:24 -060080static inline int
81xfs_buf_is_vmapped(
82 struct xfs_buf *bp)
83{
84 /*
85 * Return true if the buffer is vmapped.
86 *
87 * The XBF_MAPPED flag is set if the buffer should be mapped, but the
88 * code is clever enough to know it doesn't have to map a single page,
89 * so the check has to be both for XBF_MAPPED and bp->b_page_count > 1.
90 */
91 return (bp->b_flags & XBF_MAPPED) && bp->b_page_count > 1;
92}
93
94static inline int
95xfs_buf_vmap_len(
96 struct xfs_buf *bp)
97{
98 return (bp->b_page_count * PAGE_SIZE) - bp->b_offset;
99}
100
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100102 * Page Region interfaces.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103 *
Nathan Scottce8e9222006-01-11 15:39:08 +1100104 * For pages in filesystems where the blocksize is smaller than the
105 * pagesize, we use the page->private field (long) to hold a bitmap
106 * of uptodate regions within the page.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107 *
Nathan Scottce8e9222006-01-11 15:39:08 +1100108 * Each such region is "bytes per page / bits per long" bytes long.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109 *
Nathan Scottce8e9222006-01-11 15:39:08 +1100110 * NBPPR == number-of-bytes-per-page-region
111 * BTOPR == bytes-to-page-region (rounded up)
112 * BTOPRT == bytes-to-page-region-truncated (rounded down)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113 */
114#if (BITS_PER_LONG == 32)
115#define PRSHIFT (PAGE_CACHE_SHIFT - 5) /* (32 == 1<<5) */
116#elif (BITS_PER_LONG == 64)
117#define PRSHIFT (PAGE_CACHE_SHIFT - 6) /* (64 == 1<<6) */
118#else
119#error BITS_PER_LONG must be 32 or 64
120#endif
121#define NBPPR (PAGE_CACHE_SIZE/BITS_PER_LONG)
122#define BTOPR(b) (((unsigned int)(b) + (NBPPR - 1)) >> PRSHIFT)
123#define BTOPRT(b) (((unsigned int)(b) >> PRSHIFT))
124
125STATIC unsigned long
126page_region_mask(
127 size_t offset,
128 size_t length)
129{
130 unsigned long mask;
131 int first, final;
132
133 first = BTOPR(offset);
134 final = BTOPRT(offset + length - 1);
135 first = min(first, final);
136
137 mask = ~0UL;
138 mask <<= BITS_PER_LONG - (final - first);
139 mask >>= BITS_PER_LONG - (final);
140
141 ASSERT(offset + length <= PAGE_CACHE_SIZE);
142 ASSERT((final - first) < BITS_PER_LONG && (final - first) >= 0);
143
144 return mask;
145}
146
Christoph Hellwigb8f82a42009-11-14 16:17:22 +0000147STATIC void
Linus Torvalds1da177e2005-04-16 15:20:36 -0700148set_page_region(
149 struct page *page,
150 size_t offset,
151 size_t length)
152{
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700153 set_page_private(page,
154 page_private(page) | page_region_mask(offset, length));
155 if (page_private(page) == ~0UL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156 SetPageUptodate(page);
157}
158
Christoph Hellwigb8f82a42009-11-14 16:17:22 +0000159STATIC int
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160test_page_region(
161 struct page *page,
162 size_t offset,
163 size_t length)
164{
165 unsigned long mask = page_region_mask(offset, length);
166
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700167 return (mask && (page_private(page) & mask) == mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168}
169
170/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100171 * Internal xfs_buf_t object manipulation
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172 */
173
174STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +1100175_xfs_buf_initialize(
176 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177 xfs_buftarg_t *target,
Nathan Scott204ab252006-01-11 20:50:22 +1100178 xfs_off_t range_base,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179 size_t range_length,
Nathan Scottce8e9222006-01-11 15:39:08 +1100180 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181{
182 /*
Nathan Scottce8e9222006-01-11 15:39:08 +1100183 * We don't want certain flags to appear in b_flags.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100185 flags &= ~(XBF_LOCK|XBF_MAPPED|XBF_DONT_BLOCK|XBF_READ_AHEAD);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700186
Nathan Scottce8e9222006-01-11 15:39:08 +1100187 memset(bp, 0, sizeof(xfs_buf_t));
188 atomic_set(&bp->b_hold, 1);
David Chinnerb4dd3302008-08-13 16:36:11 +1000189 init_completion(&bp->b_iowait);
Nathan Scottce8e9222006-01-11 15:39:08 +1100190 INIT_LIST_HEAD(&bp->b_list);
Dave Chinner74f75a02010-09-24 19:59:04 +1000191 RB_CLEAR_NODE(&bp->b_rbnode);
Thomas Gleixnera731cd12010-09-07 14:33:15 +0000192 sema_init(&bp->b_sema, 0); /* held, no waiters */
Nathan Scottce8e9222006-01-11 15:39:08 +1100193 XB_SET_OWNER(bp);
194 bp->b_target = target;
195 bp->b_file_offset = range_base;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196 /*
197 * Set buffer_length and count_desired to the same value initially.
198 * I/O routines should use count_desired, which will be the same in
199 * most cases but may be reset (e.g. XFS recovery).
200 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100201 bp->b_buffer_length = bp->b_count_desired = range_length;
202 bp->b_flags = flags;
203 bp->b_bn = XFS_BUF_DADDR_NULL;
204 atomic_set(&bp->b_pin_count, 0);
205 init_waitqueue_head(&bp->b_waiters);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700206
Nathan Scottce8e9222006-01-11 15:39:08 +1100207 XFS_STATS_INC(xb_create);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000208
209 trace_xfs_buf_init(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210}
211
212/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100213 * Allocate a page array capable of holding a specified number
214 * of pages, and point the page buf at it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700215 */
216STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100217_xfs_buf_get_pages(
218 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219 int page_count,
Nathan Scottce8e9222006-01-11 15:39:08 +1100220 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221{
222 /* Make sure that we have a page list */
Nathan Scottce8e9222006-01-11 15:39:08 +1100223 if (bp->b_pages == NULL) {
224 bp->b_offset = xfs_buf_poff(bp->b_file_offset);
225 bp->b_page_count = page_count;
226 if (page_count <= XB_PAGES) {
227 bp->b_pages = bp->b_page_array;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100229 bp->b_pages = kmem_alloc(sizeof(struct page *) *
230 page_count, xb_to_km(flags));
231 if (bp->b_pages == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700232 return -ENOMEM;
233 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100234 memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700235 }
236 return 0;
237}
238
239/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100240 * Frees b_pages if it was allocated.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700241 */
242STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +1100243_xfs_buf_free_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244 xfs_buf_t *bp)
245{
Nathan Scottce8e9222006-01-11 15:39:08 +1100246 if (bp->b_pages != bp->b_page_array) {
Denys Vlasenkof0e2d932008-05-19 16:31:57 +1000247 kmem_free(bp->b_pages);
Dave Chinner3fc98b12009-12-14 23:11:57 +0000248 bp->b_pages = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249 }
250}
251
252/*
253 * Releases the specified buffer.
254 *
255 * The modification state of any associated pages is left unchanged.
Nathan Scottce8e9222006-01-11 15:39:08 +1100256 * The buffer most not be on any hash - use xfs_buf_rele instead for
Linus Torvalds1da177e2005-04-16 15:20:36 -0700257 * hashed and refcounted buffers
258 */
259void
Nathan Scottce8e9222006-01-11 15:39:08 +1100260xfs_buf_free(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700261 xfs_buf_t *bp)
262{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000263 trace_xfs_buf_free(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700264
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000265 if (bp->b_flags & (_XBF_PAGE_CACHE|_XBF_PAGES)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266 uint i;
267
James Bottomley73c77e22010-01-25 11:42:24 -0600268 if (xfs_buf_is_vmapped(bp))
Alex Elder8a262e52010-03-16 18:55:56 +0000269 vm_unmap_ram(bp->b_addr - bp->b_offset,
270 bp->b_page_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271
Nathan Scott948ecdb2006-09-28 11:03:13 +1000272 for (i = 0; i < bp->b_page_count; i++) {
273 struct page *page = bp->b_pages[i];
274
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000275 if (bp->b_flags & _XBF_PAGE_CACHE)
276 ASSERT(!PagePrivate(page));
Nathan Scott948ecdb2006-09-28 11:03:13 +1000277 page_cache_release(page);
278 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279 }
Dave Chinner3fc98b12009-12-14 23:11:57 +0000280 _xfs_buf_free_pages(bp);
Nathan Scottce8e9222006-01-11 15:39:08 +1100281 xfs_buf_deallocate(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282}
283
284/*
285 * Finds all pages for buffer in question and builds it's page list.
286 */
287STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100288_xfs_buf_lookup_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700289 xfs_buf_t *bp,
290 uint flags)
291{
Nathan Scottce8e9222006-01-11 15:39:08 +1100292 struct address_space *mapping = bp->b_target->bt_mapping;
293 size_t blocksize = bp->b_target->bt_bsize;
294 size_t size = bp->b_count_desired;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700295 size_t nbytes, offset;
Nathan Scottce8e9222006-01-11 15:39:08 +1100296 gfp_t gfp_mask = xb_to_gfp(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297 unsigned short page_count, i;
298 pgoff_t first;
Nathan Scott204ab252006-01-11 20:50:22 +1100299 xfs_off_t end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300 int error;
301
Nathan Scottce8e9222006-01-11 15:39:08 +1100302 end = bp->b_file_offset + bp->b_buffer_length;
303 page_count = xfs_buf_btoc(end) - xfs_buf_btoct(bp->b_file_offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700304
Nathan Scottce8e9222006-01-11 15:39:08 +1100305 error = _xfs_buf_get_pages(bp, page_count, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306 if (unlikely(error))
307 return error;
Nathan Scottce8e9222006-01-11 15:39:08 +1100308 bp->b_flags |= _XBF_PAGE_CACHE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309
Nathan Scottce8e9222006-01-11 15:39:08 +1100310 offset = bp->b_offset;
311 first = bp->b_file_offset >> PAGE_CACHE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312
Nathan Scottce8e9222006-01-11 15:39:08 +1100313 for (i = 0; i < bp->b_page_count; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314 struct page *page;
315 uint retries = 0;
316
317 retry:
318 page = find_or_create_page(mapping, first + i, gfp_mask);
319 if (unlikely(page == NULL)) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100320 if (flags & XBF_READ_AHEAD) {
321 bp->b_page_count = i;
Christoph Hellwig6ab455e2008-05-19 16:34:42 +1000322 for (i = 0; i < bp->b_page_count; i++)
323 unlock_page(bp->b_pages[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700324 return -ENOMEM;
325 }
326
327 /*
328 * This could deadlock.
329 *
330 * But until all the XFS lowlevel code is revamped to
331 * handle buffer allocation failures we can't do much.
332 */
333 if (!(++retries % 100))
334 printk(KERN_ERR
335 "XFS: possible memory allocation "
336 "deadlock in %s (mode:0x%x)\n",
Harvey Harrison34a622b2008-04-10 12:19:21 +1000337 __func__, gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700338
Nathan Scottce8e9222006-01-11 15:39:08 +1100339 XFS_STATS_INC(xb_page_retries);
Dave Chinner7f8275d2010-07-19 14:56:17 +1000340 xfsbufd_wakeup(NULL, 0, gfp_mask);
Jens Axboe8aa7e842009-07-09 14:52:32 +0200341 congestion_wait(BLK_RW_ASYNC, HZ/50);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700342 goto retry;
343 }
344
Nathan Scottce8e9222006-01-11 15:39:08 +1100345 XFS_STATS_INC(xb_page_found);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346
347 nbytes = min_t(size_t, size, PAGE_CACHE_SIZE - offset);
348 size -= nbytes;
349
Nathan Scott948ecdb2006-09-28 11:03:13 +1000350 ASSERT(!PagePrivate(page));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700351 if (!PageUptodate(page)) {
352 page_count--;
Christoph Hellwig6ab455e2008-05-19 16:34:42 +1000353 if (blocksize >= PAGE_CACHE_SIZE) {
354 if (flags & XBF_READ)
355 bp->b_flags |= _XBF_PAGE_LOCKED;
356 } else if (!PagePrivate(page)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357 if (test_page_region(page, offset, nbytes))
358 page_count++;
359 }
360 }
361
Nathan Scottce8e9222006-01-11 15:39:08 +1100362 bp->b_pages[i] = page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363 offset = 0;
364 }
365
Christoph Hellwig6ab455e2008-05-19 16:34:42 +1000366 if (!(bp->b_flags & _XBF_PAGE_LOCKED)) {
367 for (i = 0; i < bp->b_page_count; i++)
368 unlock_page(bp->b_pages[i]);
369 }
370
Nathan Scottce8e9222006-01-11 15:39:08 +1100371 if (page_count == bp->b_page_count)
372 bp->b_flags |= XBF_DONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374 return error;
375}
376
377/*
378 * Map buffer into kernel address-space if nessecary.
379 */
380STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100381_xfs_buf_map_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382 xfs_buf_t *bp,
383 uint flags)
384{
385 /* A single page buffer is always mappable */
Nathan Scottce8e9222006-01-11 15:39:08 +1100386 if (bp->b_page_count == 1) {
387 bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
388 bp->b_flags |= XBF_MAPPED;
389 } else if (flags & XBF_MAPPED) {
Alex Elder8a262e52010-03-16 18:55:56 +0000390 bp->b_addr = vm_map_ram(bp->b_pages, bp->b_page_count,
391 -1, PAGE_KERNEL);
Nathan Scottce8e9222006-01-11 15:39:08 +1100392 if (unlikely(bp->b_addr == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393 return -ENOMEM;
Nathan Scottce8e9222006-01-11 15:39:08 +1100394 bp->b_addr += bp->b_offset;
395 bp->b_flags |= XBF_MAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396 }
397
398 return 0;
399}
400
401/*
402 * Finding and Reading Buffers
403 */
404
405/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100406 * Look up, and creates if absent, a lockable buffer for
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407 * a given range of an inode. The buffer is returned
408 * locked. If other overlapping buffers exist, they are
409 * released before the new buffer is created and locked,
410 * which may imply that this call will block until those buffers
411 * are unlocked. No I/O is implied by this call.
412 */
413xfs_buf_t *
Nathan Scottce8e9222006-01-11 15:39:08 +1100414_xfs_buf_find(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415 xfs_buftarg_t *btp, /* block device target */
Nathan Scott204ab252006-01-11 20:50:22 +1100416 xfs_off_t ioff, /* starting offset of range */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417 size_t isize, /* length of range */
Nathan Scottce8e9222006-01-11 15:39:08 +1100418 xfs_buf_flags_t flags,
419 xfs_buf_t *new_bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420{
Nathan Scott204ab252006-01-11 20:50:22 +1100421 xfs_off_t range_base;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422 size_t range_length;
Dave Chinner74f75a02010-09-24 19:59:04 +1000423 struct xfs_perag *pag;
424 struct rb_node **rbp;
425 struct rb_node *parent;
426 xfs_buf_t *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427
428 range_base = (ioff << BBSHIFT);
429 range_length = (isize << BBSHIFT);
430
431 /* Check for IOs smaller than the sector size / not sector aligned */
Nathan Scottce8e9222006-01-11 15:39:08 +1100432 ASSERT(!(range_length < (1 << btp->bt_sshift)));
Nathan Scott204ab252006-01-11 20:50:22 +1100433 ASSERT(!(range_base & (xfs_off_t)btp->bt_smask));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434
Dave Chinner74f75a02010-09-24 19:59:04 +1000435 /* get tree root */
436 pag = xfs_perag_get(btp->bt_mount,
437 xfs_daddr_to_agno(btp->bt_mount, ioff));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700438
Dave Chinner74f75a02010-09-24 19:59:04 +1000439 /* walk tree */
440 spin_lock(&pag->pag_buf_lock);
441 rbp = &pag->pag_buf_tree.rb_node;
442 parent = NULL;
443 bp = NULL;
444 while (*rbp) {
445 parent = *rbp;
446 bp = rb_entry(parent, struct xfs_buf, b_rbnode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447
Dave Chinner74f75a02010-09-24 19:59:04 +1000448 if (range_base < bp->b_file_offset)
449 rbp = &(*rbp)->rb_left;
450 else if (range_base > bp->b_file_offset)
451 rbp = &(*rbp)->rb_right;
452 else {
453 /*
454 * found a block offset match. If the range doesn't
455 * match, the only way this is allowed is if the buffer
456 * in the cache is stale and the transaction that made
457 * it stale has not yet committed. i.e. we are
458 * reallocating a busy extent. Skip this buffer and
459 * continue searching to the right for an exact match.
460 */
461 if (bp->b_buffer_length != range_length) {
462 ASSERT(bp->b_flags & XBF_STALE);
463 rbp = &(*rbp)->rb_right;
464 continue;
465 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100466 atomic_inc(&bp->b_hold);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467 goto found;
468 }
469 }
470
471 /* No match found */
Nathan Scottce8e9222006-01-11 15:39:08 +1100472 if (new_bp) {
473 _xfs_buf_initialize(new_bp, btp, range_base,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474 range_length, flags);
Dave Chinner74f75a02010-09-24 19:59:04 +1000475 rb_link_node(&new_bp->b_rbnode, parent, rbp);
476 rb_insert_color(&new_bp->b_rbnode, &pag->pag_buf_tree);
477 /* the buffer keeps the perag reference until it is freed */
478 new_bp->b_pag = pag;
479 spin_unlock(&pag->pag_buf_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700480 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100481 XFS_STATS_INC(xb_miss_locked);
Dave Chinner74f75a02010-09-24 19:59:04 +1000482 spin_unlock(&pag->pag_buf_lock);
483 xfs_perag_put(pag);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700484 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100485 return new_bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486
487found:
Dave Chinner74f75a02010-09-24 19:59:04 +1000488 spin_unlock(&pag->pag_buf_lock);
489 xfs_perag_put(pag);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700490
491 /* Attempt to get the semaphore without sleeping,
492 * if this does not work then we need to drop the
493 * spinlock and do a hard attempt on the semaphore.
494 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100495 if (down_trylock(&bp->b_sema)) {
496 if (!(flags & XBF_TRYLOCK)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497 /* wait for buffer ownership */
Nathan Scottce8e9222006-01-11 15:39:08 +1100498 xfs_buf_lock(bp);
499 XFS_STATS_INC(xb_get_locked_waited);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500 } else {
501 /* We asked for a trylock and failed, no need
502 * to look at file offset and length here, we
Nathan Scottce8e9222006-01-11 15:39:08 +1100503 * know that this buffer at least overlaps our
504 * buffer and is locked, therefore our buffer
505 * either does not exist, or is this buffer.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700506 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100507 xfs_buf_rele(bp);
508 XFS_STATS_INC(xb_busy_locked);
509 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510 }
511 } else {
512 /* trylock worked */
Nathan Scottce8e9222006-01-11 15:39:08 +1100513 XB_SET_OWNER(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700514 }
515
Nathan Scottce8e9222006-01-11 15:39:08 +1100516 if (bp->b_flags & XBF_STALE) {
517 ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
518 bp->b_flags &= XBF_MAPPED;
David Chinner2f926582005-09-05 08:33:35 +1000519 }
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000520
521 trace_xfs_buf_find(bp, flags, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +1100522 XFS_STATS_INC(xb_get_locked);
523 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524}
525
526/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100527 * Assembles a buffer covering the specified range.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528 * Storage in memory for all portions of the buffer will be allocated,
529 * although backing storage may not be.
530 */
531xfs_buf_t *
Christoph Hellwig6ad112b2009-11-24 18:02:23 +0000532xfs_buf_get(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700533 xfs_buftarg_t *target,/* target for buffer */
Nathan Scott204ab252006-01-11 20:50:22 +1100534 xfs_off_t ioff, /* starting offset of range */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535 size_t isize, /* length of range */
Nathan Scottce8e9222006-01-11 15:39:08 +1100536 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537{
Nathan Scottce8e9222006-01-11 15:39:08 +1100538 xfs_buf_t *bp, *new_bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700539 int error = 0, i;
540
Nathan Scottce8e9222006-01-11 15:39:08 +1100541 new_bp = xfs_buf_allocate(flags);
542 if (unlikely(!new_bp))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543 return NULL;
544
Nathan Scottce8e9222006-01-11 15:39:08 +1100545 bp = _xfs_buf_find(target, ioff, isize, flags, new_bp);
546 if (bp == new_bp) {
547 error = _xfs_buf_lookup_pages(bp, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700548 if (error)
549 goto no_buffer;
550 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100551 xfs_buf_deallocate(new_bp);
552 if (unlikely(bp == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700553 return NULL;
554 }
555
Nathan Scottce8e9222006-01-11 15:39:08 +1100556 for (i = 0; i < bp->b_page_count; i++)
557 mark_page_accessed(bp->b_pages[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700558
Nathan Scottce8e9222006-01-11 15:39:08 +1100559 if (!(bp->b_flags & XBF_MAPPED)) {
560 error = _xfs_buf_map_pages(bp, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700561 if (unlikely(error)) {
562 printk(KERN_WARNING "%s: failed to map pages\n",
Harvey Harrison34a622b2008-04-10 12:19:21 +1000563 __func__);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700564 goto no_buffer;
565 }
566 }
567
Nathan Scottce8e9222006-01-11 15:39:08 +1100568 XFS_STATS_INC(xb_get);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569
570 /*
571 * Always fill in the block number now, the mapped cases can do
572 * their own overlay of this later.
573 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100574 bp->b_bn = ioff;
575 bp->b_count_desired = bp->b_buffer_length;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000577 trace_xfs_buf_get(bp, flags, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +1100578 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700579
580 no_buffer:
Nathan Scottce8e9222006-01-11 15:39:08 +1100581 if (flags & (XBF_LOCK | XBF_TRYLOCK))
582 xfs_buf_unlock(bp);
583 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584 return NULL;
585}
586
Christoph Hellwig5d765b92008-12-03 12:20:26 +0100587STATIC int
588_xfs_buf_read(
589 xfs_buf_t *bp,
590 xfs_buf_flags_t flags)
591{
592 int status;
593
Christoph Hellwig5d765b92008-12-03 12:20:26 +0100594 ASSERT(!(flags & (XBF_DELWRI|XBF_WRITE)));
595 ASSERT(bp->b_bn != XFS_BUF_DADDR_NULL);
596
597 bp->b_flags &= ~(XBF_WRITE | XBF_ASYNC | XBF_DELWRI | \
598 XBF_READ_AHEAD | _XBF_RUN_QUEUES);
599 bp->b_flags |= flags & (XBF_READ | XBF_ASYNC | \
600 XBF_READ_AHEAD | _XBF_RUN_QUEUES);
601
602 status = xfs_buf_iorequest(bp);
Dave Chinnerec53d1d2010-07-20 17:52:59 +1000603 if (status || XFS_BUF_ISERROR(bp) || (flags & XBF_ASYNC))
604 return status;
605 return xfs_buf_iowait(bp);
Christoph Hellwig5d765b92008-12-03 12:20:26 +0100606}
607
Linus Torvalds1da177e2005-04-16 15:20:36 -0700608xfs_buf_t *
Christoph Hellwig6ad112b2009-11-24 18:02:23 +0000609xfs_buf_read(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700610 xfs_buftarg_t *target,
Nathan Scott204ab252006-01-11 20:50:22 +1100611 xfs_off_t ioff,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700612 size_t isize,
Nathan Scottce8e9222006-01-11 15:39:08 +1100613 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700614{
Nathan Scottce8e9222006-01-11 15:39:08 +1100615 xfs_buf_t *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616
Nathan Scottce8e9222006-01-11 15:39:08 +1100617 flags |= XBF_READ;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700618
Christoph Hellwig6ad112b2009-11-24 18:02:23 +0000619 bp = xfs_buf_get(target, ioff, isize, flags);
Nathan Scottce8e9222006-01-11 15:39:08 +1100620 if (bp) {
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000621 trace_xfs_buf_read(bp, flags, _RET_IP_);
622
Nathan Scottce8e9222006-01-11 15:39:08 +1100623 if (!XFS_BUF_ISDONE(bp)) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100624 XFS_STATS_INC(xb_get_read);
Christoph Hellwig5d765b92008-12-03 12:20:26 +0100625 _xfs_buf_read(bp, flags);
Nathan Scottce8e9222006-01-11 15:39:08 +1100626 } else if (flags & XBF_ASYNC) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627 /*
628 * Read ahead call which is already satisfied,
629 * drop the buffer
630 */
631 goto no_buffer;
632 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700633 /* We do not want read in the flags */
Nathan Scottce8e9222006-01-11 15:39:08 +1100634 bp->b_flags &= ~XBF_READ;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700635 }
636 }
637
Nathan Scottce8e9222006-01-11 15:39:08 +1100638 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700639
640 no_buffer:
Nathan Scottce8e9222006-01-11 15:39:08 +1100641 if (flags & (XBF_LOCK | XBF_TRYLOCK))
642 xfs_buf_unlock(bp);
643 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644 return NULL;
645}
646
647/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100648 * If we are not low on memory then do the readahead in a deadlock
649 * safe manner.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700650 */
651void
Nathan Scottce8e9222006-01-11 15:39:08 +1100652xfs_buf_readahead(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700653 xfs_buftarg_t *target,
Nathan Scott204ab252006-01-11 20:50:22 +1100654 xfs_off_t ioff,
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +0000655 size_t isize)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700656{
657 struct backing_dev_info *bdi;
658
Nathan Scottce8e9222006-01-11 15:39:08 +1100659 bdi = target->bt_mapping->backing_dev_info;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700660 if (bdi_read_congested(bdi))
661 return;
662
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +0000663 xfs_buf_read(target, ioff, isize,
664 XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD|XBF_DONT_BLOCK);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700665}
666
Dave Chinner5adc94c2010-09-24 21:58:31 +1000667/*
668 * Read an uncached buffer from disk. Allocates and returns a locked
669 * buffer containing the disk contents or nothing.
670 */
671struct xfs_buf *
672xfs_buf_read_uncached(
673 struct xfs_mount *mp,
674 struct xfs_buftarg *target,
675 xfs_daddr_t daddr,
676 size_t length,
677 int flags)
678{
679 xfs_buf_t *bp;
680 int error;
681
682 bp = xfs_buf_get_uncached(target, length, flags);
683 if (!bp)
684 return NULL;
685
686 /* set up the buffer for a read IO */
687 xfs_buf_lock(bp);
688 XFS_BUF_SET_ADDR(bp, daddr);
689 XFS_BUF_READ(bp);
690 XFS_BUF_BUSY(bp);
691
692 xfsbdstrat(mp, bp);
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +0000693 error = xfs_buf_iowait(bp);
Dave Chinner5adc94c2010-09-24 21:58:31 +1000694 if (error || bp->b_error) {
695 xfs_buf_relse(bp);
696 return NULL;
697 }
698 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700699}
700
701xfs_buf_t *
Nathan Scottce8e9222006-01-11 15:39:08 +1100702xfs_buf_get_empty(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703 size_t len,
704 xfs_buftarg_t *target)
705{
Nathan Scottce8e9222006-01-11 15:39:08 +1100706 xfs_buf_t *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700707
Nathan Scottce8e9222006-01-11 15:39:08 +1100708 bp = xfs_buf_allocate(0);
709 if (bp)
710 _xfs_buf_initialize(bp, target, 0, len, 0);
711 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700712}
713
714static inline struct page *
715mem_to_page(
716 void *addr)
717{
Christoph Lameter9e2779f2008-02-04 22:28:34 -0800718 if ((!is_vmalloc_addr(addr))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700719 return virt_to_page(addr);
720 } else {
721 return vmalloc_to_page(addr);
722 }
723}
724
725int
Nathan Scottce8e9222006-01-11 15:39:08 +1100726xfs_buf_associate_memory(
727 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700728 void *mem,
729 size_t len)
730{
731 int rval;
732 int i = 0;
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100733 unsigned long pageaddr;
734 unsigned long offset;
735 size_t buflen;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736 int page_count;
737
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100738 pageaddr = (unsigned long)mem & PAGE_CACHE_MASK;
739 offset = (unsigned long)mem - pageaddr;
740 buflen = PAGE_CACHE_ALIGN(len + offset);
741 page_count = buflen >> PAGE_CACHE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700742
743 /* Free any previous set of page pointers */
Nathan Scottce8e9222006-01-11 15:39:08 +1100744 if (bp->b_pages)
745 _xfs_buf_free_pages(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746
Nathan Scottce8e9222006-01-11 15:39:08 +1100747 bp->b_pages = NULL;
748 bp->b_addr = mem;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700749
Christoph Hellwig36fae172009-07-18 18:14:58 -0400750 rval = _xfs_buf_get_pages(bp, page_count, XBF_DONT_BLOCK);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700751 if (rval)
752 return rval;
753
Nathan Scottce8e9222006-01-11 15:39:08 +1100754 bp->b_offset = offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700755
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100756 for (i = 0; i < bp->b_page_count; i++) {
757 bp->b_pages[i] = mem_to_page((void *)pageaddr);
758 pageaddr += PAGE_CACHE_SIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700759 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700760
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100761 bp->b_count_desired = len;
762 bp->b_buffer_length = buflen;
Nathan Scottce8e9222006-01-11 15:39:08 +1100763 bp->b_flags |= XBF_MAPPED;
Christoph Hellwig6ab455e2008-05-19 16:34:42 +1000764 bp->b_flags &= ~_XBF_PAGE_LOCKED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700765
766 return 0;
767}
768
769xfs_buf_t *
Dave Chinner686865f2010-09-24 20:07:47 +1000770xfs_buf_get_uncached(
771 struct xfs_buftarg *target,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700772 size_t len,
Dave Chinner686865f2010-09-24 20:07:47 +1000773 int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774{
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000775 unsigned long page_count = PAGE_ALIGN(len) >> PAGE_SHIFT;
776 int error, i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700777 xfs_buf_t *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700778
Nathan Scottce8e9222006-01-11 15:39:08 +1100779 bp = xfs_buf_allocate(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700780 if (unlikely(bp == NULL))
781 goto fail;
Nathan Scottce8e9222006-01-11 15:39:08 +1100782 _xfs_buf_initialize(bp, target, 0, len, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700783
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000784 error = _xfs_buf_get_pages(bp, page_count, 0);
785 if (error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700786 goto fail_free_buf;
787
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000788 for (i = 0; i < page_count; i++) {
Dave Chinner686865f2010-09-24 20:07:47 +1000789 bp->b_pages[i] = alloc_page(xb_to_gfp(flags));
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000790 if (!bp->b_pages[i])
791 goto fail_free_mem;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700792 }
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000793 bp->b_flags |= _XBF_PAGES;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700794
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000795 error = _xfs_buf_map_pages(bp, XBF_MAPPED);
796 if (unlikely(error)) {
797 printk(KERN_WARNING "%s: failed to map pages\n",
Harvey Harrison34a622b2008-04-10 12:19:21 +1000798 __func__);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799 goto fail_free_mem;
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000800 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700801
Nathan Scottce8e9222006-01-11 15:39:08 +1100802 xfs_buf_unlock(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700803
Dave Chinner686865f2010-09-24 20:07:47 +1000804 trace_xfs_buf_get_uncached(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700805 return bp;
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000806
Linus Torvalds1da177e2005-04-16 15:20:36 -0700807 fail_free_mem:
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000808 while (--i >= 0)
809 __free_page(bp->b_pages[i]);
Christoph Hellwigca165b82007-05-24 15:21:11 +1000810 _xfs_buf_free_pages(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700811 fail_free_buf:
Christoph Hellwigca165b82007-05-24 15:21:11 +1000812 xfs_buf_deallocate(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700813 fail:
814 return NULL;
815}
816
817/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700818 * Increment reference count on buffer, to hold the buffer concurrently
819 * with another thread which may release (free) the buffer asynchronously.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700820 * Must hold the buffer already to call this function.
821 */
822void
Nathan Scottce8e9222006-01-11 15:39:08 +1100823xfs_buf_hold(
824 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700825{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000826 trace_xfs_buf_hold(bp, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +1100827 atomic_inc(&bp->b_hold);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700828}
829
830/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100831 * Releases a hold on the specified buffer. If the
832 * the hold count is 1, calls xfs_buf_free.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700833 */
834void
Nathan Scottce8e9222006-01-11 15:39:08 +1100835xfs_buf_rele(
836 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700837{
Dave Chinner74f75a02010-09-24 19:59:04 +1000838 struct xfs_perag *pag = bp->b_pag;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700839
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000840 trace_xfs_buf_rele(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700841
Dave Chinner74f75a02010-09-24 19:59:04 +1000842 if (!pag) {
Nathan Scottfad3aa12006-02-01 12:14:52 +1100843 ASSERT(!bp->b_relse);
Dave Chinner74f75a02010-09-24 19:59:04 +1000844 ASSERT(RB_EMPTY_NODE(&bp->b_rbnode));
Nathan Scottfad3aa12006-02-01 12:14:52 +1100845 if (atomic_dec_and_test(&bp->b_hold))
846 xfs_buf_free(bp);
847 return;
848 }
849
Dave Chinner74f75a02010-09-24 19:59:04 +1000850 ASSERT(!RB_EMPTY_NODE(&bp->b_rbnode));
Lachlan McIlroy37906892008-08-13 15:42:10 +1000851 ASSERT(atomic_read(&bp->b_hold) > 0);
Dave Chinner74f75a02010-09-24 19:59:04 +1000852 if (atomic_dec_and_lock(&bp->b_hold, &pag->pag_buf_lock)) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100853 if (bp->b_relse) {
854 atomic_inc(&bp->b_hold);
Dave Chinner74f75a02010-09-24 19:59:04 +1000855 spin_unlock(&pag->pag_buf_lock);
856 bp->b_relse(bp);
Christoph Hellwig7f14d0a2005-11-02 15:09:35 +1100857 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100858 ASSERT(!(bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)));
Dave Chinner74f75a02010-09-24 19:59:04 +1000859 rb_erase(&bp->b_rbnode, &pag->pag_buf_tree);
860 spin_unlock(&pag->pag_buf_lock);
861 xfs_perag_put(pag);
Nathan Scottce8e9222006-01-11 15:39:08 +1100862 xfs_buf_free(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700863 }
864 }
865}
866
867
868/*
869 * Mutual exclusion on buffers. Locking model:
870 *
871 * Buffers associated with inodes for which buffer locking
872 * is not enabled are not protected by semaphores, and are
873 * assumed to be exclusively owned by the caller. There is a
874 * spinlock in the buffer, used by the caller when concurrent
875 * access is possible.
876 */
877
878/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100879 * Locks a buffer object, if it is not already locked.
880 * Note that this in no way locks the underlying pages, so it is only
881 * useful for synchronizing concurrent use of buffer objects, not for
882 * synchronizing independent access to the underlying pages.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700883 */
884int
Nathan Scottce8e9222006-01-11 15:39:08 +1100885xfs_buf_cond_lock(
886 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700887{
888 int locked;
889
Nathan Scottce8e9222006-01-11 15:39:08 +1100890 locked = down_trylock(&bp->b_sema) == 0;
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000891 if (locked)
Nathan Scottce8e9222006-01-11 15:39:08 +1100892 XB_SET_OWNER(bp);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000893
894 trace_xfs_buf_cond_lock(bp, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +1100895 return locked ? 0 : -EBUSY;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700896}
897
Linus Torvalds1da177e2005-04-16 15:20:36 -0700898int
Nathan Scottce8e9222006-01-11 15:39:08 +1100899xfs_buf_lock_value(
900 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901{
Stephen Rothwelladaa6932008-04-22 15:26:13 +1000902 return bp->b_sema.count;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700904
905/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100906 * Locks a buffer object.
907 * Note that this in no way locks the underlying pages, so it is only
908 * useful for synchronizing concurrent use of buffer objects, not for
909 * synchronizing independent access to the underlying pages.
Dave Chinnered3b4d62010-05-21 12:07:08 +1000910 *
911 * If we come across a stale, pinned, locked buffer, we know that we
912 * are being asked to lock a buffer that has been reallocated. Because
913 * it is pinned, we know that the log has not been pushed to disk and
914 * hence it will still be locked. Rather than sleeping until someone
915 * else pushes the log, push it ourselves before trying to get the lock.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700916 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100917void
918xfs_buf_lock(
919 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700920{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000921 trace_xfs_buf_lock(bp, _RET_IP_);
922
Dave Chinnered3b4d62010-05-21 12:07:08 +1000923 if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE))
Dave Chinnerebad8612010-09-22 10:47:20 +1000924 xfs_log_force(bp->b_target->bt_mount, 0);
Nathan Scottce8e9222006-01-11 15:39:08 +1100925 if (atomic_read(&bp->b_io_remaining))
926 blk_run_address_space(bp->b_target->bt_mapping);
927 down(&bp->b_sema);
928 XB_SET_OWNER(bp);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000929
930 trace_xfs_buf_lock_done(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700931}
932
933/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100934 * Releases the lock on the buffer object.
David Chinner2f926582005-09-05 08:33:35 +1000935 * If the buffer is marked delwri but is not queued, do so before we
Nathan Scottce8e9222006-01-11 15:39:08 +1100936 * unlock the buffer as we need to set flags correctly. We also need to
David Chinner2f926582005-09-05 08:33:35 +1000937 * take a reference for the delwri queue because the unlocker is going to
938 * drop their's and they don't know we just queued it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700939 */
940void
Nathan Scottce8e9222006-01-11 15:39:08 +1100941xfs_buf_unlock(
942 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700943{
Nathan Scottce8e9222006-01-11 15:39:08 +1100944 if ((bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)) == XBF_DELWRI) {
945 atomic_inc(&bp->b_hold);
946 bp->b_flags |= XBF_ASYNC;
947 xfs_buf_delwri_queue(bp, 0);
David Chinner2f926582005-09-05 08:33:35 +1000948 }
949
Nathan Scottce8e9222006-01-11 15:39:08 +1100950 XB_CLEAR_OWNER(bp);
951 up(&bp->b_sema);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000952
953 trace_xfs_buf_unlock(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700954}
955
Nathan Scottce8e9222006-01-11 15:39:08 +1100956STATIC void
957xfs_buf_wait_unpin(
958 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959{
960 DECLARE_WAITQUEUE (wait, current);
961
Nathan Scottce8e9222006-01-11 15:39:08 +1100962 if (atomic_read(&bp->b_pin_count) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700963 return;
964
Nathan Scottce8e9222006-01-11 15:39:08 +1100965 add_wait_queue(&bp->b_waiters, &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700966 for (;;) {
967 set_current_state(TASK_UNINTERRUPTIBLE);
Nathan Scottce8e9222006-01-11 15:39:08 +1100968 if (atomic_read(&bp->b_pin_count) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969 break;
Nathan Scottce8e9222006-01-11 15:39:08 +1100970 if (atomic_read(&bp->b_io_remaining))
971 blk_run_address_space(bp->b_target->bt_mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700972 schedule();
973 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100974 remove_wait_queue(&bp->b_waiters, &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700975 set_current_state(TASK_RUNNING);
976}
977
978/*
979 * Buffer Utility Routines
980 */
981
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +1100983xfs_buf_iodone_work(
David Howellsc4028952006-11-22 14:57:56 +0000984 struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700985{
David Howellsc4028952006-11-22 14:57:56 +0000986 xfs_buf_t *bp =
987 container_of(work, xfs_buf_t, b_iodone_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700988
Christoph Hellwig80f6c292010-08-18 05:29:11 -0400989 if (bp->b_iodone)
Nathan Scottce8e9222006-01-11 15:39:08 +1100990 (*(bp->b_iodone))(bp);
991 else if (bp->b_flags & XBF_ASYNC)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700992 xfs_buf_relse(bp);
993}
994
995void
Nathan Scottce8e9222006-01-11 15:39:08 +1100996xfs_buf_ioend(
997 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700998 int schedule)
999{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001000 trace_xfs_buf_iodone(bp, _RET_IP_);
1001
Lachlan McIlroy77be55a2007-11-23 16:31:00 +11001002 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
Nathan Scottce8e9222006-01-11 15:39:08 +11001003 if (bp->b_error == 0)
1004 bp->b_flags |= XBF_DONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001005
Nathan Scottce8e9222006-01-11 15:39:08 +11001006 if ((bp->b_iodone) || (bp->b_flags & XBF_ASYNC)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001007 if (schedule) {
David Howellsc4028952006-11-22 14:57:56 +00001008 INIT_WORK(&bp->b_iodone_work, xfs_buf_iodone_work);
Nathan Scottce8e9222006-01-11 15:39:08 +11001009 queue_work(xfslogd_workqueue, &bp->b_iodone_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001010 } else {
David Howellsc4028952006-11-22 14:57:56 +00001011 xfs_buf_iodone_work(&bp->b_iodone_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001012 }
1013 } else {
David Chinnerb4dd3302008-08-13 16:36:11 +10001014 complete(&bp->b_iowait);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001015 }
1016}
1017
Linus Torvalds1da177e2005-04-16 15:20:36 -07001018void
Nathan Scottce8e9222006-01-11 15:39:08 +11001019xfs_buf_ioerror(
1020 xfs_buf_t *bp,
1021 int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001022{
1023 ASSERT(error >= 0 && error <= 0xffff);
Nathan Scottce8e9222006-01-11 15:39:08 +11001024 bp->b_error = (unsigned short)error;
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001025 trace_xfs_buf_ioerror(bp, error, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001026}
1027
Linus Torvalds1da177e2005-04-16 15:20:36 -07001028int
Christoph Hellwig64e0bc72010-01-13 22:17:58 +00001029xfs_bwrite(
1030 struct xfs_mount *mp,
Christoph Hellwig5d765b92008-12-03 12:20:26 +01001031 struct xfs_buf *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001032{
Christoph Hellwig8c383662010-03-12 10:59:40 +00001033 int error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001034
Christoph Hellwig64e0bc72010-01-13 22:17:58 +00001035 bp->b_flags |= XBF_WRITE;
Christoph Hellwig8c383662010-03-12 10:59:40 +00001036 bp->b_flags &= ~(XBF_ASYNC | XBF_READ);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001037
Christoph Hellwig5d765b92008-12-03 12:20:26 +01001038 xfs_buf_delwri_dequeue(bp);
Christoph Hellwig939d7232010-07-20 17:51:16 +10001039 xfs_bdstrat_cb(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001040
Christoph Hellwig8c383662010-03-12 10:59:40 +00001041 error = xfs_buf_iowait(bp);
1042 if (error)
1043 xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
1044 xfs_buf_relse(bp);
Christoph Hellwig64e0bc72010-01-13 22:17:58 +00001045 return error;
Christoph Hellwig5d765b92008-12-03 12:20:26 +01001046}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001047
Christoph Hellwig5d765b92008-12-03 12:20:26 +01001048void
1049xfs_bdwrite(
1050 void *mp,
1051 struct xfs_buf *bp)
1052{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001053 trace_xfs_buf_bdwrite(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001054
Christoph Hellwig5d765b92008-12-03 12:20:26 +01001055 bp->b_flags &= ~XBF_READ;
1056 bp->b_flags |= (XBF_DELWRI | XBF_ASYNC);
1057
1058 xfs_buf_delwri_queue(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001059}
1060
Christoph Hellwig4e234712010-01-13 22:17:56 +00001061/*
1062 * Called when we want to stop a buffer from getting written or read.
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +00001063 * We attach the EIO error, muck with its flags, and call xfs_buf_ioend
Christoph Hellwig4e234712010-01-13 22:17:56 +00001064 * so that the proper iodone callbacks get called.
1065 */
1066STATIC int
1067xfs_bioerror(
1068 xfs_buf_t *bp)
1069{
1070#ifdef XFSERRORDEBUG
1071 ASSERT(XFS_BUF_ISREAD(bp) || bp->b_iodone);
1072#endif
1073
1074 /*
1075 * No need to wait until the buffer is unpinned, we aren't flushing it.
1076 */
1077 XFS_BUF_ERROR(bp, EIO);
1078
1079 /*
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +00001080 * We're calling xfs_buf_ioend, so delete XBF_DONE flag.
Christoph Hellwig4e234712010-01-13 22:17:56 +00001081 */
1082 XFS_BUF_UNREAD(bp);
1083 XFS_BUF_UNDELAYWRITE(bp);
1084 XFS_BUF_UNDONE(bp);
1085 XFS_BUF_STALE(bp);
1086
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +00001087 xfs_buf_ioend(bp, 0);
Christoph Hellwig4e234712010-01-13 22:17:56 +00001088
1089 return EIO;
1090}
1091
1092/*
1093 * Same as xfs_bioerror, except that we are releasing the buffer
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +00001094 * here ourselves, and avoiding the xfs_buf_ioend call.
Christoph Hellwig4e234712010-01-13 22:17:56 +00001095 * This is meant for userdata errors; metadata bufs come with
1096 * iodone functions attached, so that we can track down errors.
1097 */
1098STATIC int
1099xfs_bioerror_relse(
1100 struct xfs_buf *bp)
1101{
1102 int64_t fl = XFS_BUF_BFLAGS(bp);
1103 /*
1104 * No need to wait until the buffer is unpinned.
1105 * We aren't flushing it.
1106 *
1107 * chunkhold expects B_DONE to be set, whether
1108 * we actually finish the I/O or not. We don't want to
1109 * change that interface.
1110 */
1111 XFS_BUF_UNREAD(bp);
1112 XFS_BUF_UNDELAYWRITE(bp);
1113 XFS_BUF_DONE(bp);
1114 XFS_BUF_STALE(bp);
1115 XFS_BUF_CLR_IODONE_FUNC(bp);
Christoph Hellwig0cadda12010-01-19 09:56:44 +00001116 if (!(fl & XBF_ASYNC)) {
Christoph Hellwig4e234712010-01-13 22:17:56 +00001117 /*
1118 * Mark b_error and B_ERROR _both_.
1119 * Lot's of chunkcache code assumes that.
1120 * There's no reason to mark error for
1121 * ASYNC buffers.
1122 */
1123 XFS_BUF_ERROR(bp, EIO);
1124 XFS_BUF_FINISH_IOWAIT(bp);
1125 } else {
1126 xfs_buf_relse(bp);
1127 }
1128
1129 return EIO;
1130}
1131
1132
1133/*
1134 * All xfs metadata buffers except log state machine buffers
1135 * get this attached as their b_bdstrat callback function.
1136 * This is so that we can catch a buffer
1137 * after prematurely unpinning it to forcibly shutdown the filesystem.
1138 */
1139int
1140xfs_bdstrat_cb(
1141 struct xfs_buf *bp)
1142{
Dave Chinnerebad8612010-09-22 10:47:20 +10001143 if (XFS_FORCED_SHUTDOWN(bp->b_target->bt_mount)) {
Christoph Hellwig4e234712010-01-13 22:17:56 +00001144 trace_xfs_bdstrat_shut(bp, _RET_IP_);
1145 /*
1146 * Metadata write that didn't get logged but
1147 * written delayed anyway. These aren't associated
1148 * with a transaction, and can be ignored.
1149 */
1150 if (!bp->b_iodone && !XFS_BUF_ISREAD(bp))
1151 return xfs_bioerror_relse(bp);
1152 else
1153 return xfs_bioerror(bp);
1154 }
1155
1156 xfs_buf_iorequest(bp);
1157 return 0;
1158}
1159
1160/*
1161 * Wrapper around bdstrat so that we can stop data from going to disk in case
1162 * we are shutting down the filesystem. Typically user data goes thru this
1163 * path; one of the exceptions is the superblock.
1164 */
1165void
1166xfsbdstrat(
1167 struct xfs_mount *mp,
1168 struct xfs_buf *bp)
1169{
1170 if (XFS_FORCED_SHUTDOWN(mp)) {
1171 trace_xfs_bdstrat_shut(bp, _RET_IP_);
1172 xfs_bioerror_relse(bp);
1173 return;
1174 }
1175
1176 xfs_buf_iorequest(bp);
1177}
1178
Christoph Hellwigb8f82a42009-11-14 16:17:22 +00001179STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001180_xfs_buf_ioend(
1181 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001182 int schedule)
1183{
Christoph Hellwig6ab455e2008-05-19 16:34:42 +10001184 if (atomic_dec_and_test(&bp->b_io_remaining) == 1) {
1185 bp->b_flags &= ~_XBF_PAGE_LOCKED;
Nathan Scottce8e9222006-01-11 15:39:08 +11001186 xfs_buf_ioend(bp, schedule);
Christoph Hellwig6ab455e2008-05-19 16:34:42 +10001187 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001188}
1189
Al Viro782e3b32007-10-12 07:17:47 +01001190STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001191xfs_buf_bio_end_io(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001192 struct bio *bio,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001193 int error)
1194{
Nathan Scottce8e9222006-01-11 15:39:08 +11001195 xfs_buf_t *bp = (xfs_buf_t *)bio->bi_private;
1196 unsigned int blocksize = bp->b_target->bt_bsize;
Nathan Scotteedb5532005-09-02 16:39:56 +10001197 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001198
Lachlan McIlroycfbe5262008-12-12 15:27:25 +11001199 xfs_buf_ioerror(bp, -error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001200
James Bottomley73c77e22010-01-25 11:42:24 -06001201 if (!error && xfs_buf_is_vmapped(bp) && (bp->b_flags & XBF_READ))
1202 invalidate_kernel_vmap_range(bp->b_addr, xfs_buf_vmap_len(bp));
1203
Nathan Scotteedb5532005-09-02 16:39:56 +10001204 do {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001205 struct page *page = bvec->bv_page;
1206
Nathan Scott948ecdb2006-09-28 11:03:13 +10001207 ASSERT(!PagePrivate(page));
Nathan Scottce8e9222006-01-11 15:39:08 +11001208 if (unlikely(bp->b_error)) {
1209 if (bp->b_flags & XBF_READ)
Nathan Scotteedb5532005-09-02 16:39:56 +10001210 ClearPageUptodate(page);
Nathan Scottce8e9222006-01-11 15:39:08 +11001211 } else if (blocksize >= PAGE_CACHE_SIZE) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001212 SetPageUptodate(page);
1213 } else if (!PagePrivate(page) &&
Nathan Scottce8e9222006-01-11 15:39:08 +11001214 (bp->b_flags & _XBF_PAGE_CACHE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001215 set_page_region(page, bvec->bv_offset, bvec->bv_len);
1216 }
1217
Nathan Scotteedb5532005-09-02 16:39:56 +10001218 if (--bvec >= bio->bi_io_vec)
1219 prefetchw(&bvec->bv_page->flags);
Christoph Hellwig6ab455e2008-05-19 16:34:42 +10001220
1221 if (bp->b_flags & _XBF_PAGE_LOCKED)
1222 unlock_page(page);
Nathan Scotteedb5532005-09-02 16:39:56 +10001223 } while (bvec >= bio->bi_io_vec);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001224
Nathan Scottce8e9222006-01-11 15:39:08 +11001225 _xfs_buf_ioend(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001226 bio_put(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001227}
1228
1229STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001230_xfs_buf_ioapply(
1231 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001232{
Christoph Hellwiga9759f22007-12-07 14:07:08 +11001233 int rw, map_i, total_nr_pages, nr_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001234 struct bio *bio;
Nathan Scottce8e9222006-01-11 15:39:08 +11001235 int offset = bp->b_offset;
1236 int size = bp->b_count_desired;
1237 sector_t sector = bp->b_bn;
1238 unsigned int blocksize = bp->b_target->bt_bsize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001239
Nathan Scottce8e9222006-01-11 15:39:08 +11001240 total_nr_pages = bp->b_page_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001241 map_i = 0;
1242
Nathan Scottce8e9222006-01-11 15:39:08 +11001243 if (bp->b_flags & XBF_ORDERED) {
1244 ASSERT(!(bp->b_flags & XBF_READ));
Christoph Hellwig80f6c292010-08-18 05:29:11 -04001245 rw = WRITE_FLUSH_FUA;
Dave Chinner2ee1aba2009-11-24 18:03:15 +00001246 } else if (bp->b_flags & XBF_LOG_BUFFER) {
Nathan Scott51bdd702006-09-28 11:01:57 +10001247 ASSERT(!(bp->b_flags & XBF_READ_AHEAD));
1248 bp->b_flags &= ~_XBF_RUN_QUEUES;
1249 rw = (bp->b_flags & XBF_WRITE) ? WRITE_SYNC : READ_SYNC;
Dave Chinner2ee1aba2009-11-24 18:03:15 +00001250 } else if (bp->b_flags & _XBF_RUN_QUEUES) {
1251 ASSERT(!(bp->b_flags & XBF_READ_AHEAD));
1252 bp->b_flags &= ~_XBF_RUN_QUEUES;
1253 rw = (bp->b_flags & XBF_WRITE) ? WRITE_META : READ_META;
Nathan Scott51bdd702006-09-28 11:01:57 +10001254 } else {
1255 rw = (bp->b_flags & XBF_WRITE) ? WRITE :
1256 (bp->b_flags & XBF_READ_AHEAD) ? READA : READ;
Christoph Hellwigf538d4d2005-11-02 10:26:59 +11001257 }
1258
Nathan Scottce8e9222006-01-11 15:39:08 +11001259 /* Special code path for reading a sub page size buffer in --
Linus Torvalds1da177e2005-04-16 15:20:36 -07001260 * we populate up the whole page, and hence the other metadata
1261 * in the same page. This optimization is only valid when the
Nathan Scottce8e9222006-01-11 15:39:08 +11001262 * filesystem block size is not smaller than the page size.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001263 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001264 if ((bp->b_buffer_length < PAGE_CACHE_SIZE) &&
Christoph Hellwig6ab455e2008-05-19 16:34:42 +10001265 ((bp->b_flags & (XBF_READ|_XBF_PAGE_LOCKED)) ==
1266 (XBF_READ|_XBF_PAGE_LOCKED)) &&
Nathan Scottce8e9222006-01-11 15:39:08 +11001267 (blocksize >= PAGE_CACHE_SIZE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001268 bio = bio_alloc(GFP_NOIO, 1);
1269
Nathan Scottce8e9222006-01-11 15:39:08 +11001270 bio->bi_bdev = bp->b_target->bt_bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001271 bio->bi_sector = sector - (offset >> BBSHIFT);
Nathan Scottce8e9222006-01-11 15:39:08 +11001272 bio->bi_end_io = xfs_buf_bio_end_io;
1273 bio->bi_private = bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001274
Nathan Scottce8e9222006-01-11 15:39:08 +11001275 bio_add_page(bio, bp->b_pages[0], PAGE_CACHE_SIZE, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001276 size = 0;
1277
Nathan Scottce8e9222006-01-11 15:39:08 +11001278 atomic_inc(&bp->b_io_remaining);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001279
1280 goto submit_io;
1281 }
1282
Linus Torvalds1da177e2005-04-16 15:20:36 -07001283next_chunk:
Nathan Scottce8e9222006-01-11 15:39:08 +11001284 atomic_inc(&bp->b_io_remaining);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001285 nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1286 if (nr_pages > total_nr_pages)
1287 nr_pages = total_nr_pages;
1288
1289 bio = bio_alloc(GFP_NOIO, nr_pages);
Nathan Scottce8e9222006-01-11 15:39:08 +11001290 bio->bi_bdev = bp->b_target->bt_bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001291 bio->bi_sector = sector;
Nathan Scottce8e9222006-01-11 15:39:08 +11001292 bio->bi_end_io = xfs_buf_bio_end_io;
1293 bio->bi_private = bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001294
1295 for (; size && nr_pages; nr_pages--, map_i++) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001296 int rbytes, nbytes = PAGE_CACHE_SIZE - offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001297
1298 if (nbytes > size)
1299 nbytes = size;
1300
Nathan Scottce8e9222006-01-11 15:39:08 +11001301 rbytes = bio_add_page(bio, bp->b_pages[map_i], nbytes, offset);
1302 if (rbytes < nbytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001303 break;
1304
1305 offset = 0;
1306 sector += nbytes >> BBSHIFT;
1307 size -= nbytes;
1308 total_nr_pages--;
1309 }
1310
1311submit_io:
1312 if (likely(bio->bi_size)) {
James Bottomley73c77e22010-01-25 11:42:24 -06001313 if (xfs_buf_is_vmapped(bp)) {
1314 flush_kernel_vmap_range(bp->b_addr,
1315 xfs_buf_vmap_len(bp));
1316 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001317 submit_bio(rw, bio);
1318 if (size)
1319 goto next_chunk;
1320 } else {
Dave Chinnerec53d1d2010-07-20 17:52:59 +10001321 /*
1322 * if we get here, no pages were added to the bio. However,
1323 * we can't just error out here - if the pages are locked then
1324 * we have to unlock them otherwise we can hang on a later
1325 * access to the page.
1326 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001327 xfs_buf_ioerror(bp, EIO);
Dave Chinnerec53d1d2010-07-20 17:52:59 +10001328 if (bp->b_flags & _XBF_PAGE_LOCKED) {
1329 int i;
1330 for (i = 0; i < bp->b_page_count; i++)
1331 unlock_page(bp->b_pages[i]);
1332 }
1333 bio_put(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001334 }
1335}
1336
Linus Torvalds1da177e2005-04-16 15:20:36 -07001337int
Nathan Scottce8e9222006-01-11 15:39:08 +11001338xfs_buf_iorequest(
1339 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001340{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001341 trace_xfs_buf_iorequest(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001342
Nathan Scottce8e9222006-01-11 15:39:08 +11001343 if (bp->b_flags & XBF_DELWRI) {
1344 xfs_buf_delwri_queue(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001345 return 0;
1346 }
1347
Nathan Scottce8e9222006-01-11 15:39:08 +11001348 if (bp->b_flags & XBF_WRITE) {
1349 xfs_buf_wait_unpin(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001350 }
1351
Nathan Scottce8e9222006-01-11 15:39:08 +11001352 xfs_buf_hold(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001353
1354 /* Set the count to 1 initially, this will stop an I/O
1355 * completion callout which happens before we have started
Nathan Scottce8e9222006-01-11 15:39:08 +11001356 * all the I/O from calling xfs_buf_ioend too early.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001357 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001358 atomic_set(&bp->b_io_remaining, 1);
1359 _xfs_buf_ioapply(bp);
1360 _xfs_buf_ioend(bp, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001361
Nathan Scottce8e9222006-01-11 15:39:08 +11001362 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001363 return 0;
1364}
1365
1366/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001367 * Waits for I/O to complete on the buffer supplied.
1368 * It returns immediately if no I/O is pending.
1369 * It returns the I/O error code, if any, or 0 if there was no error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001370 */
1371int
Nathan Scottce8e9222006-01-11 15:39:08 +11001372xfs_buf_iowait(
1373 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001374{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001375 trace_xfs_buf_iowait(bp, _RET_IP_);
1376
Nathan Scottce8e9222006-01-11 15:39:08 +11001377 if (atomic_read(&bp->b_io_remaining))
1378 blk_run_address_space(bp->b_target->bt_mapping);
David Chinnerb4dd3302008-08-13 16:36:11 +10001379 wait_for_completion(&bp->b_iowait);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001380
1381 trace_xfs_buf_iowait_done(bp, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +11001382 return bp->b_error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001383}
1384
Nathan Scottce8e9222006-01-11 15:39:08 +11001385xfs_caddr_t
1386xfs_buf_offset(
1387 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001388 size_t offset)
1389{
1390 struct page *page;
1391
Nathan Scottce8e9222006-01-11 15:39:08 +11001392 if (bp->b_flags & XBF_MAPPED)
1393 return XFS_BUF_PTR(bp) + offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001394
Nathan Scottce8e9222006-01-11 15:39:08 +11001395 offset += bp->b_offset;
1396 page = bp->b_pages[offset >> PAGE_CACHE_SHIFT];
1397 return (xfs_caddr_t)page_address(page) + (offset & (PAGE_CACHE_SIZE-1));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001398}
1399
1400/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001401 * Move data into or out of a buffer.
1402 */
1403void
Nathan Scottce8e9222006-01-11 15:39:08 +11001404xfs_buf_iomove(
1405 xfs_buf_t *bp, /* buffer to process */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001406 size_t boff, /* starting buffer offset */
1407 size_t bsize, /* length to copy */
Dave Chinnerb9c48642010-01-20 10:47:39 +11001408 void *data, /* data address */
Nathan Scottce8e9222006-01-11 15:39:08 +11001409 xfs_buf_rw_t mode) /* read/write/zero flag */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001410{
1411 size_t bend, cpoff, csize;
1412 struct page *page;
1413
1414 bend = boff + bsize;
1415 while (boff < bend) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001416 page = bp->b_pages[xfs_buf_btoct(boff + bp->b_offset)];
1417 cpoff = xfs_buf_poff(boff + bp->b_offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001418 csize = min_t(size_t,
Nathan Scottce8e9222006-01-11 15:39:08 +11001419 PAGE_CACHE_SIZE-cpoff, bp->b_count_desired-boff);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001420
1421 ASSERT(((csize + cpoff) <= PAGE_CACHE_SIZE));
1422
1423 switch (mode) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001424 case XBRW_ZERO:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001425 memset(page_address(page) + cpoff, 0, csize);
1426 break;
Nathan Scottce8e9222006-01-11 15:39:08 +11001427 case XBRW_READ:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001428 memcpy(data, page_address(page) + cpoff, csize);
1429 break;
Nathan Scottce8e9222006-01-11 15:39:08 +11001430 case XBRW_WRITE:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001431 memcpy(page_address(page) + cpoff, data, csize);
1432 }
1433
1434 boff += csize;
1435 data += csize;
1436 }
1437}
1438
1439/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001440 * Handling of buffer targets (buftargs).
Linus Torvalds1da177e2005-04-16 15:20:36 -07001441 */
1442
1443/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001444 * Wait for any bufs with callbacks that have been submitted but
1445 * have not yet returned... walk the hash list for the target.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001446 */
1447void
1448xfs_wait_buftarg(
Dave Chinner74f75a02010-09-24 19:59:04 +10001449 struct xfs_buftarg *btp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001450{
Dave Chinner74f75a02010-09-24 19:59:04 +10001451 struct xfs_perag *pag;
1452 uint i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001453
Dave Chinner74f75a02010-09-24 19:59:04 +10001454 for (i = 0; i < btp->bt_mount->m_sb.sb_agcount; i++) {
1455 pag = xfs_perag_get(btp->bt_mount, i);
1456 spin_lock(&pag->pag_buf_lock);
1457 while (rb_first(&pag->pag_buf_tree)) {
1458 spin_unlock(&pag->pag_buf_lock);
Dave Chinner26af6552010-09-22 10:47:20 +10001459 delay(100);
Dave Chinner74f75a02010-09-24 19:59:04 +10001460 spin_lock(&pag->pag_buf_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001461 }
Dave Chinner74f75a02010-09-24 19:59:04 +10001462 spin_unlock(&pag->pag_buf_lock);
1463 xfs_perag_put(pag);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001464 }
1465}
1466
1467/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001468 * buftarg list for delwrite queue processing
David Chinnera6867a62006-01-11 15:37:58 +11001469 */
Tim Shimmine6a0e9c2007-05-08 13:49:59 +10001470static LIST_HEAD(xfs_buftarg_list);
David Chinner7989cb82007-02-10 18:34:56 +11001471static DEFINE_SPINLOCK(xfs_buftarg_lock);
David Chinnera6867a62006-01-11 15:37:58 +11001472
1473STATIC void
1474xfs_register_buftarg(
1475 xfs_buftarg_t *btp)
1476{
1477 spin_lock(&xfs_buftarg_lock);
1478 list_add(&btp->bt_list, &xfs_buftarg_list);
1479 spin_unlock(&xfs_buftarg_lock);
1480}
1481
1482STATIC void
1483xfs_unregister_buftarg(
1484 xfs_buftarg_t *btp)
1485{
1486 spin_lock(&xfs_buftarg_lock);
1487 list_del(&btp->bt_list);
1488 spin_unlock(&xfs_buftarg_lock);
1489}
1490
Linus Torvalds1da177e2005-04-16 15:20:36 -07001491void
1492xfs_free_buftarg(
Christoph Hellwigb7963132009-03-03 14:48:37 -05001493 struct xfs_mount *mp,
1494 struct xfs_buftarg *btp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001495{
1496 xfs_flush_buftarg(btp, 1);
Christoph Hellwigb7963132009-03-03 14:48:37 -05001497 if (mp->m_flags & XFS_MOUNT_BARRIER)
1498 xfs_blkdev_issue_flush(btp);
Nathan Scottce8e9222006-01-11 15:39:08 +11001499 iput(btp->bt_mapping->host);
David Chinnera6867a62006-01-11 15:37:58 +11001500
Nathan Scottce8e9222006-01-11 15:39:08 +11001501 /* Unregister the buftarg first so that we don't get a
1502 * wakeup finding a non-existent task
1503 */
David Chinnera6867a62006-01-11 15:37:58 +11001504 xfs_unregister_buftarg(btp);
1505 kthread_stop(btp->bt_task);
1506
Denys Vlasenkof0e2d932008-05-19 16:31:57 +10001507 kmem_free(btp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001508}
1509
Linus Torvalds1da177e2005-04-16 15:20:36 -07001510STATIC int
1511xfs_setsize_buftarg_flags(
1512 xfs_buftarg_t *btp,
1513 unsigned int blocksize,
1514 unsigned int sectorsize,
1515 int verbose)
1516{
Nathan Scottce8e9222006-01-11 15:39:08 +11001517 btp->bt_bsize = blocksize;
1518 btp->bt_sshift = ffs(sectorsize) - 1;
1519 btp->bt_smask = sectorsize - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001520
Nathan Scottce8e9222006-01-11 15:39:08 +11001521 if (set_blocksize(btp->bt_bdev, sectorsize)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001522 printk(KERN_WARNING
1523 "XFS: Cannot set_blocksize to %u on device %s\n",
1524 sectorsize, XFS_BUFTARG_NAME(btp));
1525 return EINVAL;
1526 }
1527
1528 if (verbose &&
1529 (PAGE_CACHE_SIZE / BITS_PER_LONG) > sectorsize) {
1530 printk(KERN_WARNING
1531 "XFS: %u byte sectors in use on device %s. "
1532 "This is suboptimal; %u or greater is ideal.\n",
1533 sectorsize, XFS_BUFTARG_NAME(btp),
1534 (unsigned int)PAGE_CACHE_SIZE / BITS_PER_LONG);
1535 }
1536
1537 return 0;
1538}
1539
1540/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001541 * When allocating the initial buffer target we have not yet
1542 * read in the superblock, so don't know what sized sectors
1543 * are being used is at this early stage. Play safe.
1544 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001545STATIC int
1546xfs_setsize_buftarg_early(
1547 xfs_buftarg_t *btp,
1548 struct block_device *bdev)
1549{
1550 return xfs_setsize_buftarg_flags(btp,
Martin K. Petersene1defc42009-05-22 17:17:49 -04001551 PAGE_CACHE_SIZE, bdev_logical_block_size(bdev), 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001552}
1553
1554int
1555xfs_setsize_buftarg(
1556 xfs_buftarg_t *btp,
1557 unsigned int blocksize,
1558 unsigned int sectorsize)
1559{
1560 return xfs_setsize_buftarg_flags(btp, blocksize, sectorsize, 1);
1561}
1562
1563STATIC int
1564xfs_mapping_buftarg(
1565 xfs_buftarg_t *btp,
1566 struct block_device *bdev)
1567{
1568 struct backing_dev_info *bdi;
1569 struct inode *inode;
1570 struct address_space *mapping;
Christoph Hellwigf5e54d62006-06-28 04:26:44 -07001571 static const struct address_space_operations mapping_aops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001572 .sync_page = block_sync_page,
Christoph Lametere965f962006-02-01 03:05:41 -08001573 .migratepage = fail_migrate_page,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001574 };
1575
1576 inode = new_inode(bdev->bd_inode->i_sb);
1577 if (!inode) {
1578 printk(KERN_WARNING
1579 "XFS: Cannot allocate mapping inode for device %s\n",
1580 XFS_BUFTARG_NAME(btp));
1581 return ENOMEM;
1582 }
Christoph Hellwig85fe4022010-10-23 11:19:54 -04001583 inode->i_ino = get_next_ino();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001584 inode->i_mode = S_IFBLK;
1585 inode->i_bdev = bdev;
1586 inode->i_rdev = bdev->bd_dev;
1587 bdi = blk_get_backing_dev_info(bdev);
1588 if (!bdi)
1589 bdi = &default_backing_dev_info;
1590 mapping = &inode->i_data;
1591 mapping->a_ops = &mapping_aops;
1592 mapping->backing_dev_info = bdi;
1593 mapping_set_gfp_mask(mapping, GFP_NOFS);
Nathan Scottce8e9222006-01-11 15:39:08 +11001594 btp->bt_mapping = mapping;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001595 return 0;
1596}
1597
David Chinnera6867a62006-01-11 15:37:58 +11001598STATIC int
1599xfs_alloc_delwrite_queue(
Jan Engelhardte2a07812010-03-23 09:52:55 +11001600 xfs_buftarg_t *btp,
1601 const char *fsname)
David Chinnera6867a62006-01-11 15:37:58 +11001602{
1603 int error = 0;
1604
1605 INIT_LIST_HEAD(&btp->bt_list);
1606 INIT_LIST_HEAD(&btp->bt_delwrite_queue);
Eric Sandeen007c61c2007-10-11 17:43:56 +10001607 spin_lock_init(&btp->bt_delwrite_lock);
David Chinnera6867a62006-01-11 15:37:58 +11001608 btp->bt_flags = 0;
Jan Engelhardte2a07812010-03-23 09:52:55 +11001609 btp->bt_task = kthread_run(xfsbufd, btp, "xfsbufd/%s", fsname);
David Chinnera6867a62006-01-11 15:37:58 +11001610 if (IS_ERR(btp->bt_task)) {
1611 error = PTR_ERR(btp->bt_task);
1612 goto out_error;
1613 }
1614 xfs_register_buftarg(btp);
1615out_error:
1616 return error;
1617}
1618
Linus Torvalds1da177e2005-04-16 15:20:36 -07001619xfs_buftarg_t *
1620xfs_alloc_buftarg(
Dave Chinnerebad8612010-09-22 10:47:20 +10001621 struct xfs_mount *mp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001622 struct block_device *bdev,
Jan Engelhardte2a07812010-03-23 09:52:55 +11001623 int external,
1624 const char *fsname)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001625{
1626 xfs_buftarg_t *btp;
1627
1628 btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
1629
Dave Chinnerebad8612010-09-22 10:47:20 +10001630 btp->bt_mount = mp;
Nathan Scottce8e9222006-01-11 15:39:08 +11001631 btp->bt_dev = bdev->bd_dev;
1632 btp->bt_bdev = bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001633 if (xfs_setsize_buftarg_early(btp, bdev))
1634 goto error;
1635 if (xfs_mapping_buftarg(btp, bdev))
1636 goto error;
Jan Engelhardte2a07812010-03-23 09:52:55 +11001637 if (xfs_alloc_delwrite_queue(btp, fsname))
David Chinnera6867a62006-01-11 15:37:58 +11001638 goto error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001639 return btp;
1640
1641error:
Denys Vlasenkof0e2d932008-05-19 16:31:57 +10001642 kmem_free(btp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001643 return NULL;
1644}
1645
1646
1647/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001648 * Delayed write buffer handling
Linus Torvalds1da177e2005-04-16 15:20:36 -07001649 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001650STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001651xfs_buf_delwri_queue(
1652 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001653 int unlock)
1654{
Nathan Scottce8e9222006-01-11 15:39:08 +11001655 struct list_head *dwq = &bp->b_target->bt_delwrite_queue;
1656 spinlock_t *dwlk = &bp->b_target->bt_delwrite_lock;
David Chinnera6867a62006-01-11 15:37:58 +11001657
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001658 trace_xfs_buf_delwri_queue(bp, _RET_IP_);
1659
Nathan Scottce8e9222006-01-11 15:39:08 +11001660 ASSERT((bp->b_flags&(XBF_DELWRI|XBF_ASYNC)) == (XBF_DELWRI|XBF_ASYNC));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001661
David Chinnera6867a62006-01-11 15:37:58 +11001662 spin_lock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001663 /* If already in the queue, dequeue and place at tail */
Nathan Scottce8e9222006-01-11 15:39:08 +11001664 if (!list_empty(&bp->b_list)) {
1665 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1666 if (unlock)
1667 atomic_dec(&bp->b_hold);
1668 list_del(&bp->b_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001669 }
1670
Dave Chinnerc9c12972010-01-11 11:49:59 +00001671 if (list_empty(dwq)) {
1672 /* start xfsbufd as it is about to have something to do */
1673 wake_up_process(bp->b_target->bt_task);
1674 }
1675
Nathan Scottce8e9222006-01-11 15:39:08 +11001676 bp->b_flags |= _XBF_DELWRI_Q;
1677 list_add_tail(&bp->b_list, dwq);
1678 bp->b_queuetime = jiffies;
David Chinnera6867a62006-01-11 15:37:58 +11001679 spin_unlock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001680
1681 if (unlock)
Nathan Scottce8e9222006-01-11 15:39:08 +11001682 xfs_buf_unlock(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001683}
1684
1685void
Nathan Scottce8e9222006-01-11 15:39:08 +11001686xfs_buf_delwri_dequeue(
1687 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001688{
Nathan Scottce8e9222006-01-11 15:39:08 +11001689 spinlock_t *dwlk = &bp->b_target->bt_delwrite_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001690 int dequeued = 0;
1691
David Chinnera6867a62006-01-11 15:37:58 +11001692 spin_lock(dwlk);
Nathan Scottce8e9222006-01-11 15:39:08 +11001693 if ((bp->b_flags & XBF_DELWRI) && !list_empty(&bp->b_list)) {
1694 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1695 list_del_init(&bp->b_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001696 dequeued = 1;
1697 }
Nathan Scottce8e9222006-01-11 15:39:08 +11001698 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
David Chinnera6867a62006-01-11 15:37:58 +11001699 spin_unlock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001700
1701 if (dequeued)
Nathan Scottce8e9222006-01-11 15:39:08 +11001702 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001703
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001704 trace_xfs_buf_delwri_dequeue(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001705}
1706
Dave Chinnerd808f612010-02-02 10:13:42 +11001707/*
1708 * If a delwri buffer needs to be pushed before it has aged out, then promote
1709 * it to the head of the delwri queue so that it will be flushed on the next
1710 * xfsbufd run. We do this by resetting the queuetime of the buffer to be older
1711 * than the age currently needed to flush the buffer. Hence the next time the
1712 * xfsbufd sees it is guaranteed to be considered old enough to flush.
1713 */
1714void
1715xfs_buf_delwri_promote(
1716 struct xfs_buf *bp)
1717{
1718 struct xfs_buftarg *btp = bp->b_target;
1719 long age = xfs_buf_age_centisecs * msecs_to_jiffies(10) + 1;
1720
1721 ASSERT(bp->b_flags & XBF_DELWRI);
1722 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1723
1724 /*
1725 * Check the buffer age before locking the delayed write queue as we
1726 * don't need to promote buffers that are already past the flush age.
1727 */
1728 if (bp->b_queuetime < jiffies - age)
1729 return;
1730 bp->b_queuetime = jiffies - age;
1731 spin_lock(&btp->bt_delwrite_lock);
1732 list_move(&bp->b_list, &btp->bt_delwrite_queue);
1733 spin_unlock(&btp->bt_delwrite_lock);
1734}
1735
Linus Torvalds1da177e2005-04-16 15:20:36 -07001736STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001737xfs_buf_runall_queues(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001738 struct workqueue_struct *queue)
1739{
1740 flush_workqueue(queue);
1741}
1742
Linus Torvalds1da177e2005-04-16 15:20:36 -07001743STATIC int
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001744xfsbufd_wakeup(
Dave Chinner7f8275d2010-07-19 14:56:17 +10001745 struct shrinker *shrink,
Nathan Scott15c84a42005-11-04 10:51:01 +11001746 int priority,
1747 gfp_t mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001748{
Christoph Hellwigda7f93e2006-01-11 20:49:57 +11001749 xfs_buftarg_t *btp;
David Chinnera6867a62006-01-11 15:37:58 +11001750
1751 spin_lock(&xfs_buftarg_lock);
Christoph Hellwigda7f93e2006-01-11 20:49:57 +11001752 list_for_each_entry(btp, &xfs_buftarg_list, bt_list) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001753 if (test_bit(XBT_FORCE_SLEEP, &btp->bt_flags))
David Chinnera6867a62006-01-11 15:37:58 +11001754 continue;
Dave Chinnerc9c12972010-01-11 11:49:59 +00001755 if (list_empty(&btp->bt_delwrite_queue))
1756 continue;
Nathan Scottce8e9222006-01-11 15:39:08 +11001757 set_bit(XBT_FORCE_FLUSH, &btp->bt_flags);
David Chinnera6867a62006-01-11 15:37:58 +11001758 wake_up_process(btp->bt_task);
1759 }
1760 spin_unlock(&xfs_buftarg_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001761 return 0;
1762}
1763
David Chinner585e6d82007-02-10 18:32:29 +11001764/*
1765 * Move as many buffers as specified to the supplied list
1766 * idicating if we skipped any buffers to prevent deadlocks.
1767 */
1768STATIC int
1769xfs_buf_delwri_split(
1770 xfs_buftarg_t *target,
1771 struct list_head *list,
David Chinner5e6a07d2007-02-10 18:34:49 +11001772 unsigned long age)
David Chinner585e6d82007-02-10 18:32:29 +11001773{
1774 xfs_buf_t *bp, *n;
1775 struct list_head *dwq = &target->bt_delwrite_queue;
1776 spinlock_t *dwlk = &target->bt_delwrite_lock;
1777 int skipped = 0;
David Chinner5e6a07d2007-02-10 18:34:49 +11001778 int force;
David Chinner585e6d82007-02-10 18:32:29 +11001779
David Chinner5e6a07d2007-02-10 18:34:49 +11001780 force = test_and_clear_bit(XBT_FORCE_FLUSH, &target->bt_flags);
David Chinner585e6d82007-02-10 18:32:29 +11001781 INIT_LIST_HEAD(list);
1782 spin_lock(dwlk);
1783 list_for_each_entry_safe(bp, n, dwq, b_list) {
David Chinner585e6d82007-02-10 18:32:29 +11001784 ASSERT(bp->b_flags & XBF_DELWRI);
1785
Christoph Hellwig4d16e922010-06-23 18:11:15 +10001786 if (!XFS_BUF_ISPINNED(bp) && !xfs_buf_cond_lock(bp)) {
David Chinner5e6a07d2007-02-10 18:34:49 +11001787 if (!force &&
David Chinner585e6d82007-02-10 18:32:29 +11001788 time_before(jiffies, bp->b_queuetime + age)) {
1789 xfs_buf_unlock(bp);
1790 break;
1791 }
1792
1793 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q|
1794 _XBF_RUN_QUEUES);
1795 bp->b_flags |= XBF_WRITE;
1796 list_move_tail(&bp->b_list, list);
Dave Chinnerbfe27412010-11-08 08:55:05 +00001797 trace_xfs_buf_delwri_split(bp, _RET_IP_);
David Chinner585e6d82007-02-10 18:32:29 +11001798 } else
1799 skipped++;
1800 }
1801 spin_unlock(dwlk);
1802
1803 return skipped;
1804
1805}
1806
Dave Chinner089716a2010-01-26 15:13:25 +11001807/*
1808 * Compare function is more complex than it needs to be because
1809 * the return value is only 32 bits and we are doing comparisons
1810 * on 64 bit values
1811 */
1812static int
1813xfs_buf_cmp(
1814 void *priv,
1815 struct list_head *a,
1816 struct list_head *b)
1817{
1818 struct xfs_buf *ap = container_of(a, struct xfs_buf, b_list);
1819 struct xfs_buf *bp = container_of(b, struct xfs_buf, b_list);
1820 xfs_daddr_t diff;
1821
1822 diff = ap->b_bn - bp->b_bn;
1823 if (diff < 0)
1824 return -1;
1825 if (diff > 0)
1826 return 1;
1827 return 0;
1828}
1829
1830void
1831xfs_buf_delwri_sort(
1832 xfs_buftarg_t *target,
1833 struct list_head *list)
1834{
1835 list_sort(NULL, list, xfs_buf_cmp);
1836}
1837
Linus Torvalds1da177e2005-04-16 15:20:36 -07001838STATIC int
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001839xfsbufd(
David Chinner585e6d82007-02-10 18:32:29 +11001840 void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001841{
Dave Chinner089716a2010-01-26 15:13:25 +11001842 xfs_buftarg_t *target = (xfs_buftarg_t *)data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001843
Linus Torvalds1da177e2005-04-16 15:20:36 -07001844 current->flags |= PF_MEMALLOC;
1845
Rafael J. Wysocki978c7b22007-12-07 14:09:02 +11001846 set_freezable();
1847
Linus Torvalds1da177e2005-04-16 15:20:36 -07001848 do {
Dave Chinnerc9c12972010-01-11 11:49:59 +00001849 long age = xfs_buf_age_centisecs * msecs_to_jiffies(10);
1850 long tout = xfs_buf_timer_centisecs * msecs_to_jiffies(10);
Dave Chinner089716a2010-01-26 15:13:25 +11001851 int count = 0;
1852 struct list_head tmp;
Dave Chinnerc9c12972010-01-11 11:49:59 +00001853
Christoph Lameter3e1d1d22005-06-24 23:13:50 -07001854 if (unlikely(freezing(current))) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001855 set_bit(XBT_FORCE_SLEEP, &target->bt_flags);
Christoph Lameter3e1d1d22005-06-24 23:13:50 -07001856 refrigerator();
Nathan Scottabd0cf72005-05-05 13:30:13 -07001857 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +11001858 clear_bit(XBT_FORCE_SLEEP, &target->bt_flags);
Nathan Scottabd0cf72005-05-05 13:30:13 -07001859 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001860
Dave Chinnerc9c12972010-01-11 11:49:59 +00001861 /* sleep for a long time if there is nothing to do. */
1862 if (list_empty(&target->bt_delwrite_queue))
1863 tout = MAX_SCHEDULE_TIMEOUT;
1864 schedule_timeout_interruptible(tout);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001865
Dave Chinnerc9c12972010-01-11 11:49:59 +00001866 xfs_buf_delwri_split(target, &tmp, age);
Dave Chinner089716a2010-01-26 15:13:25 +11001867 list_sort(NULL, &tmp, xfs_buf_cmp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001868 while (!list_empty(&tmp)) {
Dave Chinner089716a2010-01-26 15:13:25 +11001869 struct xfs_buf *bp;
1870 bp = list_first_entry(&tmp, struct xfs_buf, b_list);
Nathan Scottce8e9222006-01-11 15:39:08 +11001871 list_del_init(&bp->b_list);
Christoph Hellwig939d7232010-07-20 17:51:16 +10001872 xfs_bdstrat_cb(bp);
David Chinner585e6d82007-02-10 18:32:29 +11001873 count++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001874 }
Nathan Scottf07c2252006-09-28 10:52:15 +10001875 if (count)
1876 blk_run_address_space(target->bt_mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001877
Christoph Hellwig4df08c52005-09-05 08:34:18 +10001878 } while (!kthread_should_stop());
Linus Torvalds1da177e2005-04-16 15:20:36 -07001879
Christoph Hellwig4df08c52005-09-05 08:34:18 +10001880 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001881}
1882
1883/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001884 * Go through all incore buffers, and release buffers if they belong to
1885 * the given device. This is used in filesystem error handling to
1886 * preserve the consistency of its metadata.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001887 */
1888int
1889xfs_flush_buftarg(
David Chinner585e6d82007-02-10 18:32:29 +11001890 xfs_buftarg_t *target,
1891 int wait)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001892{
Dave Chinner089716a2010-01-26 15:13:25 +11001893 xfs_buf_t *bp;
David Chinner585e6d82007-02-10 18:32:29 +11001894 int pincount = 0;
Dave Chinner089716a2010-01-26 15:13:25 +11001895 LIST_HEAD(tmp_list);
1896 LIST_HEAD(wait_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001897
Dave Chinnerc626d172009-04-06 18:42:11 +02001898 xfs_buf_runall_queues(xfsconvertd_workqueue);
Nathan Scottce8e9222006-01-11 15:39:08 +11001899 xfs_buf_runall_queues(xfsdatad_workqueue);
1900 xfs_buf_runall_queues(xfslogd_workqueue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001901
David Chinner5e6a07d2007-02-10 18:34:49 +11001902 set_bit(XBT_FORCE_FLUSH, &target->bt_flags);
Dave Chinner089716a2010-01-26 15:13:25 +11001903 pincount = xfs_buf_delwri_split(target, &tmp_list, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001904
1905 /*
Dave Chinner089716a2010-01-26 15:13:25 +11001906 * Dropped the delayed write list lock, now walk the temporary list.
1907 * All I/O is issued async and then if we need to wait for completion
1908 * we do that after issuing all the IO.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001909 */
Dave Chinner089716a2010-01-26 15:13:25 +11001910 list_sort(NULL, &tmp_list, xfs_buf_cmp);
1911 while (!list_empty(&tmp_list)) {
1912 bp = list_first_entry(&tmp_list, struct xfs_buf, b_list);
David Chinner585e6d82007-02-10 18:32:29 +11001913 ASSERT(target == bp->b_target);
Dave Chinner089716a2010-01-26 15:13:25 +11001914 list_del_init(&bp->b_list);
1915 if (wait) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001916 bp->b_flags &= ~XBF_ASYNC;
Dave Chinner089716a2010-01-26 15:13:25 +11001917 list_add(&bp->b_list, &wait_list);
1918 }
Christoph Hellwig939d7232010-07-20 17:51:16 +10001919 xfs_bdstrat_cb(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001920 }
1921
Dave Chinner089716a2010-01-26 15:13:25 +11001922 if (wait) {
1923 /* Expedite and wait for IO to complete. */
Nathan Scottf07c2252006-09-28 10:52:15 +10001924 blk_run_address_space(target->bt_mapping);
Dave Chinner089716a2010-01-26 15:13:25 +11001925 while (!list_empty(&wait_list)) {
1926 bp = list_first_entry(&wait_list, struct xfs_buf, b_list);
Nathan Scottf07c2252006-09-28 10:52:15 +10001927
Dave Chinner089716a2010-01-26 15:13:25 +11001928 list_del_init(&bp->b_list);
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +00001929 xfs_buf_iowait(bp);
Dave Chinner089716a2010-01-26 15:13:25 +11001930 xfs_buf_relse(bp);
1931 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001932 }
1933
Linus Torvalds1da177e2005-04-16 15:20:36 -07001934 return pincount;
1935}
1936
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001937int __init
Nathan Scottce8e9222006-01-11 15:39:08 +11001938xfs_buf_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001939{
Nathan Scott87582802006-03-14 13:18:19 +11001940 xfs_buf_zone = kmem_zone_init_flags(sizeof(xfs_buf_t), "xfs_buf",
1941 KM_ZONE_HWALIGN, NULL);
Nathan Scottce8e9222006-01-11 15:39:08 +11001942 if (!xfs_buf_zone)
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001943 goto out;
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001944
Dave Chinner51749e42010-09-08 09:00:22 +00001945 xfslogd_workqueue = alloc_workqueue("xfslogd",
Tejun Heo6370a6a2010-10-11 15:12:27 +02001946 WQ_MEM_RECLAIM | WQ_HIGHPRI, 1);
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001947 if (!xfslogd_workqueue)
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001948 goto out_free_buf_zone;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001949
Rafael J. Wysockib4337692007-03-22 00:11:27 -08001950 xfsdatad_workqueue = create_workqueue("xfsdatad");
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001951 if (!xfsdatad_workqueue)
1952 goto out_destroy_xfslogd_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001953
Dave Chinnerc626d172009-04-06 18:42:11 +02001954 xfsconvertd_workqueue = create_workqueue("xfsconvertd");
1955 if (!xfsconvertd_workqueue)
1956 goto out_destroy_xfsdatad_workqueue;
1957
Rusty Russell8e1f9362007-07-17 04:03:17 -07001958 register_shrinker(&xfs_buf_shake);
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001959 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001960
Dave Chinnerc626d172009-04-06 18:42:11 +02001961 out_destroy_xfsdatad_workqueue:
1962 destroy_workqueue(xfsdatad_workqueue);
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001963 out_destroy_xfslogd_workqueue:
1964 destroy_workqueue(xfslogd_workqueue);
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001965 out_free_buf_zone:
Nathan Scottce8e9222006-01-11 15:39:08 +11001966 kmem_zone_destroy(xfs_buf_zone);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001967 out:
Nathan Scott87582802006-03-14 13:18:19 +11001968 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001969}
1970
Linus Torvalds1da177e2005-04-16 15:20:36 -07001971void
Nathan Scottce8e9222006-01-11 15:39:08 +11001972xfs_buf_terminate(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001973{
Rusty Russell8e1f9362007-07-17 04:03:17 -07001974 unregister_shrinker(&xfs_buf_shake);
Dave Chinnerc626d172009-04-06 18:42:11 +02001975 destroy_workqueue(xfsconvertd_workqueue);
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001976 destroy_workqueue(xfsdatad_workqueue);
1977 destroy_workqueue(xfslogd_workqueue);
Nathan Scottce8e9222006-01-11 15:39:08 +11001978 kmem_zone_destroy(xfs_buf_zone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001979}
Tim Shimmine6a0e9c2007-05-08 13:49:59 +10001980
1981#ifdef CONFIG_KDB_MODULES
1982struct list_head *
1983xfs_get_buftarg_list(void)
1984{
1985 return &xfs_buftarg_list;
1986}
1987#endif