blob: bd111b7e1daa5812b8b1d68907ecc390caecd8e0 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Nathan Scottf07c2252006-09-28 10:52:15 +10002 * Copyright (c) 2000-2006 Silicon Graphics, Inc.
Nathan Scott7b718762005-11-02 14:58:39 +11003 * All Rights Reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
Nathan Scott7b718762005-11-02 14:58:39 +11005 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License as
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 * published by the Free Software Foundation.
8 *
Nathan Scott7b718762005-11-02 14:58:39 +11009 * This program is distributed in the hope that it would be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 *
Nathan Scott7b718762005-11-02 14:58:39 +110014 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
Linus Torvalds1da177e2005-04-16 15:20:36 -070017 */
Vlad Apostolov93c189c2006-11-11 18:03:49 +110018#include "xfs.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/stddef.h>
20#include <linux/errno.h>
21#include <linux/slab.h>
22#include <linux/pagemap.h>
23#include <linux/init.h>
24#include <linux/vmalloc.h>
25#include <linux/bio.h>
26#include <linux/sysctl.h>
27#include <linux/proc_fs.h>
28#include <linux/workqueue.h>
29#include <linux/percpu.h>
30#include <linux/blkdev.h>
31#include <linux/hash.h>
Christoph Hellwig4df08c52005-09-05 08:34:18 +100032#include <linux/kthread.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080033#include <linux/migrate.h>
Andrew Morton3fcfab12006-10-19 23:28:16 -070034#include <linux/backing-dev.h>
Nigel Cunningham7dfb7102006-12-06 20:34:23 -080035#include <linux/freezer.h>
Dave Chinner089716a2010-01-26 15:13:25 +110036#include <linux/list_sort.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070037
Christoph Hellwigb7963132009-03-03 14:48:37 -050038#include "xfs_sb.h"
39#include "xfs_inum.h"
40#include "xfs_ag.h"
41#include "xfs_dmapi.h"
42#include "xfs_mount.h"
Christoph Hellwig0b1b2132009-12-14 23:14:59 +000043#include "xfs_trace.h"
Christoph Hellwigb7963132009-03-03 14:48:37 -050044
David Chinner7989cb82007-02-10 18:34:56 +110045static kmem_zone_t *xfs_buf_zone;
David Chinnera6867a62006-01-11 15:37:58 +110046STATIC int xfsbufd(void *);
Al Viro27496a82005-10-21 03:20:48 -040047STATIC int xfsbufd_wakeup(int, gfp_t);
Nathan Scottce8e9222006-01-11 15:39:08 +110048STATIC void xfs_buf_delwri_queue(xfs_buf_t *, int);
Rusty Russell8e1f9362007-07-17 04:03:17 -070049static struct shrinker xfs_buf_shake = {
50 .shrink = xfsbufd_wakeup,
51 .seeks = DEFAULT_SEEKS,
52};
Christoph Hellwig23ea4032005-06-21 15:14:01 +100053
David Chinner7989cb82007-02-10 18:34:56 +110054static struct workqueue_struct *xfslogd_workqueue;
Christoph Hellwig0829c362005-09-02 16:58:49 +100055struct workqueue_struct *xfsdatad_workqueue;
Dave Chinnerc626d172009-04-06 18:42:11 +020056struct workqueue_struct *xfsconvertd_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070057
Nathan Scottce8e9222006-01-11 15:39:08 +110058#ifdef XFS_BUF_LOCK_TRACKING
59# define XB_SET_OWNER(bp) ((bp)->b_last_holder = current->pid)
60# define XB_CLEAR_OWNER(bp) ((bp)->b_last_holder = -1)
61# define XB_GET_OWNER(bp) ((bp)->b_last_holder)
Linus Torvalds1da177e2005-04-16 15:20:36 -070062#else
Nathan Scottce8e9222006-01-11 15:39:08 +110063# define XB_SET_OWNER(bp) do { } while (0)
64# define XB_CLEAR_OWNER(bp) do { } while (0)
65# define XB_GET_OWNER(bp) do { } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -070066#endif
67
Nathan Scottce8e9222006-01-11 15:39:08 +110068#define xb_to_gfp(flags) \
69 ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : \
70 ((flags) & XBF_DONT_BLOCK) ? GFP_NOFS : GFP_KERNEL) | __GFP_NOWARN)
Linus Torvalds1da177e2005-04-16 15:20:36 -070071
Nathan Scottce8e9222006-01-11 15:39:08 +110072#define xb_to_km(flags) \
73 (((flags) & XBF_DONT_BLOCK) ? KM_NOFS : KM_SLEEP)
Linus Torvalds1da177e2005-04-16 15:20:36 -070074
Nathan Scottce8e9222006-01-11 15:39:08 +110075#define xfs_buf_allocate(flags) \
76 kmem_zone_alloc(xfs_buf_zone, xb_to_km(flags))
77#define xfs_buf_deallocate(bp) \
78 kmem_zone_free(xfs_buf_zone, (bp));
Linus Torvalds1da177e2005-04-16 15:20:36 -070079
James Bottomley73c77e22010-01-25 11:42:24 -060080static inline int
81xfs_buf_is_vmapped(
82 struct xfs_buf *bp)
83{
84 /*
85 * Return true if the buffer is vmapped.
86 *
87 * The XBF_MAPPED flag is set if the buffer should be mapped, but the
88 * code is clever enough to know it doesn't have to map a single page,
89 * so the check has to be both for XBF_MAPPED and bp->b_page_count > 1.
90 */
91 return (bp->b_flags & XBF_MAPPED) && bp->b_page_count > 1;
92}
93
94static inline int
95xfs_buf_vmap_len(
96 struct xfs_buf *bp)
97{
98 return (bp->b_page_count * PAGE_SIZE) - bp->b_offset;
99}
100
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100102 * Page Region interfaces.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103 *
Nathan Scottce8e9222006-01-11 15:39:08 +1100104 * For pages in filesystems where the blocksize is smaller than the
105 * pagesize, we use the page->private field (long) to hold a bitmap
106 * of uptodate regions within the page.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107 *
Nathan Scottce8e9222006-01-11 15:39:08 +1100108 * Each such region is "bytes per page / bits per long" bytes long.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109 *
Nathan Scottce8e9222006-01-11 15:39:08 +1100110 * NBPPR == number-of-bytes-per-page-region
111 * BTOPR == bytes-to-page-region (rounded up)
112 * BTOPRT == bytes-to-page-region-truncated (rounded down)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113 */
114#if (BITS_PER_LONG == 32)
115#define PRSHIFT (PAGE_CACHE_SHIFT - 5) /* (32 == 1<<5) */
116#elif (BITS_PER_LONG == 64)
117#define PRSHIFT (PAGE_CACHE_SHIFT - 6) /* (64 == 1<<6) */
118#else
119#error BITS_PER_LONG must be 32 or 64
120#endif
121#define NBPPR (PAGE_CACHE_SIZE/BITS_PER_LONG)
122#define BTOPR(b) (((unsigned int)(b) + (NBPPR - 1)) >> PRSHIFT)
123#define BTOPRT(b) (((unsigned int)(b) >> PRSHIFT))
124
125STATIC unsigned long
126page_region_mask(
127 size_t offset,
128 size_t length)
129{
130 unsigned long mask;
131 int first, final;
132
133 first = BTOPR(offset);
134 final = BTOPRT(offset + length - 1);
135 first = min(first, final);
136
137 mask = ~0UL;
138 mask <<= BITS_PER_LONG - (final - first);
139 mask >>= BITS_PER_LONG - (final);
140
141 ASSERT(offset + length <= PAGE_CACHE_SIZE);
142 ASSERT((final - first) < BITS_PER_LONG && (final - first) >= 0);
143
144 return mask;
145}
146
Christoph Hellwigb8f82a42009-11-14 16:17:22 +0000147STATIC void
Linus Torvalds1da177e2005-04-16 15:20:36 -0700148set_page_region(
149 struct page *page,
150 size_t offset,
151 size_t length)
152{
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700153 set_page_private(page,
154 page_private(page) | page_region_mask(offset, length));
155 if (page_private(page) == ~0UL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156 SetPageUptodate(page);
157}
158
Christoph Hellwigb8f82a42009-11-14 16:17:22 +0000159STATIC int
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160test_page_region(
161 struct page *page,
162 size_t offset,
163 size_t length)
164{
165 unsigned long mask = page_region_mask(offset, length);
166
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700167 return (mask && (page_private(page) & mask) == mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168}
169
170/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100171 * Internal xfs_buf_t object manipulation
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172 */
173
174STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +1100175_xfs_buf_initialize(
176 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177 xfs_buftarg_t *target,
Nathan Scott204ab252006-01-11 20:50:22 +1100178 xfs_off_t range_base,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179 size_t range_length,
Nathan Scottce8e9222006-01-11 15:39:08 +1100180 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181{
182 /*
Nathan Scottce8e9222006-01-11 15:39:08 +1100183 * We don't want certain flags to appear in b_flags.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100185 flags &= ~(XBF_LOCK|XBF_MAPPED|XBF_DONT_BLOCK|XBF_READ_AHEAD);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700186
Nathan Scottce8e9222006-01-11 15:39:08 +1100187 memset(bp, 0, sizeof(xfs_buf_t));
188 atomic_set(&bp->b_hold, 1);
David Chinnerb4dd3302008-08-13 16:36:11 +1000189 init_completion(&bp->b_iowait);
Nathan Scottce8e9222006-01-11 15:39:08 +1100190 INIT_LIST_HEAD(&bp->b_list);
191 INIT_LIST_HEAD(&bp->b_hash_list);
192 init_MUTEX_LOCKED(&bp->b_sema); /* held, no waiters */
193 XB_SET_OWNER(bp);
194 bp->b_target = target;
195 bp->b_file_offset = range_base;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196 /*
197 * Set buffer_length and count_desired to the same value initially.
198 * I/O routines should use count_desired, which will be the same in
199 * most cases but may be reset (e.g. XFS recovery).
200 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100201 bp->b_buffer_length = bp->b_count_desired = range_length;
202 bp->b_flags = flags;
203 bp->b_bn = XFS_BUF_DADDR_NULL;
204 atomic_set(&bp->b_pin_count, 0);
205 init_waitqueue_head(&bp->b_waiters);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700206
Nathan Scottce8e9222006-01-11 15:39:08 +1100207 XFS_STATS_INC(xb_create);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000208
209 trace_xfs_buf_init(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210}
211
212/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100213 * Allocate a page array capable of holding a specified number
214 * of pages, and point the page buf at it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700215 */
216STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100217_xfs_buf_get_pages(
218 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219 int page_count,
Nathan Scottce8e9222006-01-11 15:39:08 +1100220 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221{
222 /* Make sure that we have a page list */
Nathan Scottce8e9222006-01-11 15:39:08 +1100223 if (bp->b_pages == NULL) {
224 bp->b_offset = xfs_buf_poff(bp->b_file_offset);
225 bp->b_page_count = page_count;
226 if (page_count <= XB_PAGES) {
227 bp->b_pages = bp->b_page_array;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100229 bp->b_pages = kmem_alloc(sizeof(struct page *) *
230 page_count, xb_to_km(flags));
231 if (bp->b_pages == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700232 return -ENOMEM;
233 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100234 memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700235 }
236 return 0;
237}
238
239/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100240 * Frees b_pages if it was allocated.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700241 */
242STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +1100243_xfs_buf_free_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244 xfs_buf_t *bp)
245{
Nathan Scottce8e9222006-01-11 15:39:08 +1100246 if (bp->b_pages != bp->b_page_array) {
Denys Vlasenkof0e2d932008-05-19 16:31:57 +1000247 kmem_free(bp->b_pages);
Dave Chinner3fc98b12009-12-14 23:11:57 +0000248 bp->b_pages = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249 }
250}
251
252/*
253 * Releases the specified buffer.
254 *
255 * The modification state of any associated pages is left unchanged.
Nathan Scottce8e9222006-01-11 15:39:08 +1100256 * The buffer most not be on any hash - use xfs_buf_rele instead for
Linus Torvalds1da177e2005-04-16 15:20:36 -0700257 * hashed and refcounted buffers
258 */
259void
Nathan Scottce8e9222006-01-11 15:39:08 +1100260xfs_buf_free(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700261 xfs_buf_t *bp)
262{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000263 trace_xfs_buf_free(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700264
Nathan Scottce8e9222006-01-11 15:39:08 +1100265 ASSERT(list_empty(&bp->b_hash_list));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000267 if (bp->b_flags & (_XBF_PAGE_CACHE|_XBF_PAGES)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268 uint i;
269
James Bottomley73c77e22010-01-25 11:42:24 -0600270 if (xfs_buf_is_vmapped(bp))
Alex Elder8a262e52010-03-16 18:55:56 +0000271 vm_unmap_ram(bp->b_addr - bp->b_offset,
272 bp->b_page_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700273
Nathan Scott948ecdb2006-09-28 11:03:13 +1000274 for (i = 0; i < bp->b_page_count; i++) {
275 struct page *page = bp->b_pages[i];
276
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000277 if (bp->b_flags & _XBF_PAGE_CACHE)
278 ASSERT(!PagePrivate(page));
Nathan Scott948ecdb2006-09-28 11:03:13 +1000279 page_cache_release(page);
280 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700281 }
Dave Chinner3fc98b12009-12-14 23:11:57 +0000282 _xfs_buf_free_pages(bp);
Nathan Scottce8e9222006-01-11 15:39:08 +1100283 xfs_buf_deallocate(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284}
285
286/*
287 * Finds all pages for buffer in question and builds it's page list.
288 */
289STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100290_xfs_buf_lookup_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291 xfs_buf_t *bp,
292 uint flags)
293{
Nathan Scottce8e9222006-01-11 15:39:08 +1100294 struct address_space *mapping = bp->b_target->bt_mapping;
295 size_t blocksize = bp->b_target->bt_bsize;
296 size_t size = bp->b_count_desired;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297 size_t nbytes, offset;
Nathan Scottce8e9222006-01-11 15:39:08 +1100298 gfp_t gfp_mask = xb_to_gfp(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299 unsigned short page_count, i;
300 pgoff_t first;
Nathan Scott204ab252006-01-11 20:50:22 +1100301 xfs_off_t end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700302 int error;
303
Nathan Scottce8e9222006-01-11 15:39:08 +1100304 end = bp->b_file_offset + bp->b_buffer_length;
305 page_count = xfs_buf_btoc(end) - xfs_buf_btoct(bp->b_file_offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306
Nathan Scottce8e9222006-01-11 15:39:08 +1100307 error = _xfs_buf_get_pages(bp, page_count, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308 if (unlikely(error))
309 return error;
Nathan Scottce8e9222006-01-11 15:39:08 +1100310 bp->b_flags |= _XBF_PAGE_CACHE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311
Nathan Scottce8e9222006-01-11 15:39:08 +1100312 offset = bp->b_offset;
313 first = bp->b_file_offset >> PAGE_CACHE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314
Nathan Scottce8e9222006-01-11 15:39:08 +1100315 for (i = 0; i < bp->b_page_count; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700316 struct page *page;
317 uint retries = 0;
318
319 retry:
320 page = find_or_create_page(mapping, first + i, gfp_mask);
321 if (unlikely(page == NULL)) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100322 if (flags & XBF_READ_AHEAD) {
323 bp->b_page_count = i;
Christoph Hellwig6ab455e2008-05-19 16:34:42 +1000324 for (i = 0; i < bp->b_page_count; i++)
325 unlock_page(bp->b_pages[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700326 return -ENOMEM;
327 }
328
329 /*
330 * This could deadlock.
331 *
332 * But until all the XFS lowlevel code is revamped to
333 * handle buffer allocation failures we can't do much.
334 */
335 if (!(++retries % 100))
336 printk(KERN_ERR
337 "XFS: possible memory allocation "
338 "deadlock in %s (mode:0x%x)\n",
Harvey Harrison34a622b2008-04-10 12:19:21 +1000339 __func__, gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340
Nathan Scottce8e9222006-01-11 15:39:08 +1100341 XFS_STATS_INC(xb_page_retries);
Christoph Hellwig23ea4032005-06-21 15:14:01 +1000342 xfsbufd_wakeup(0, gfp_mask);
Jens Axboe8aa7e842009-07-09 14:52:32 +0200343 congestion_wait(BLK_RW_ASYNC, HZ/50);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344 goto retry;
345 }
346
Nathan Scottce8e9222006-01-11 15:39:08 +1100347 XFS_STATS_INC(xb_page_found);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348
349 nbytes = min_t(size_t, size, PAGE_CACHE_SIZE - offset);
350 size -= nbytes;
351
Nathan Scott948ecdb2006-09-28 11:03:13 +1000352 ASSERT(!PagePrivate(page));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353 if (!PageUptodate(page)) {
354 page_count--;
Christoph Hellwig6ab455e2008-05-19 16:34:42 +1000355 if (blocksize >= PAGE_CACHE_SIZE) {
356 if (flags & XBF_READ)
357 bp->b_flags |= _XBF_PAGE_LOCKED;
358 } else if (!PagePrivate(page)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700359 if (test_page_region(page, offset, nbytes))
360 page_count++;
361 }
362 }
363
Nathan Scottce8e9222006-01-11 15:39:08 +1100364 bp->b_pages[i] = page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700365 offset = 0;
366 }
367
Christoph Hellwig6ab455e2008-05-19 16:34:42 +1000368 if (!(bp->b_flags & _XBF_PAGE_LOCKED)) {
369 for (i = 0; i < bp->b_page_count; i++)
370 unlock_page(bp->b_pages[i]);
371 }
372
Nathan Scottce8e9222006-01-11 15:39:08 +1100373 if (page_count == bp->b_page_count)
374 bp->b_flags |= XBF_DONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700375
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376 return error;
377}
378
379/*
380 * Map buffer into kernel address-space if nessecary.
381 */
382STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100383_xfs_buf_map_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384 xfs_buf_t *bp,
385 uint flags)
386{
387 /* A single page buffer is always mappable */
Nathan Scottce8e9222006-01-11 15:39:08 +1100388 if (bp->b_page_count == 1) {
389 bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
390 bp->b_flags |= XBF_MAPPED;
391 } else if (flags & XBF_MAPPED) {
Alex Elder8a262e52010-03-16 18:55:56 +0000392 bp->b_addr = vm_map_ram(bp->b_pages, bp->b_page_count,
393 -1, PAGE_KERNEL);
Nathan Scottce8e9222006-01-11 15:39:08 +1100394 if (unlikely(bp->b_addr == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700395 return -ENOMEM;
Nathan Scottce8e9222006-01-11 15:39:08 +1100396 bp->b_addr += bp->b_offset;
397 bp->b_flags |= XBF_MAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398 }
399
400 return 0;
401}
402
403/*
404 * Finding and Reading Buffers
405 */
406
407/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100408 * Look up, and creates if absent, a lockable buffer for
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409 * a given range of an inode. The buffer is returned
410 * locked. If other overlapping buffers exist, they are
411 * released before the new buffer is created and locked,
412 * which may imply that this call will block until those buffers
413 * are unlocked. No I/O is implied by this call.
414 */
415xfs_buf_t *
Nathan Scottce8e9222006-01-11 15:39:08 +1100416_xfs_buf_find(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417 xfs_buftarg_t *btp, /* block device target */
Nathan Scott204ab252006-01-11 20:50:22 +1100418 xfs_off_t ioff, /* starting offset of range */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419 size_t isize, /* length of range */
Nathan Scottce8e9222006-01-11 15:39:08 +1100420 xfs_buf_flags_t flags,
421 xfs_buf_t *new_bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422{
Nathan Scott204ab252006-01-11 20:50:22 +1100423 xfs_off_t range_base;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424 size_t range_length;
425 xfs_bufhash_t *hash;
Nathan Scottce8e9222006-01-11 15:39:08 +1100426 xfs_buf_t *bp, *n;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427
428 range_base = (ioff << BBSHIFT);
429 range_length = (isize << BBSHIFT);
430
431 /* Check for IOs smaller than the sector size / not sector aligned */
Nathan Scottce8e9222006-01-11 15:39:08 +1100432 ASSERT(!(range_length < (1 << btp->bt_sshift)));
Nathan Scott204ab252006-01-11 20:50:22 +1100433 ASSERT(!(range_base & (xfs_off_t)btp->bt_smask));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434
435 hash = &btp->bt_hash[hash_long((unsigned long)ioff, btp->bt_hashshift)];
436
437 spin_lock(&hash->bh_lock);
438
Nathan Scottce8e9222006-01-11 15:39:08 +1100439 list_for_each_entry_safe(bp, n, &hash->bh_list, b_hash_list) {
440 ASSERT(btp == bp->b_target);
441 if (bp->b_file_offset == range_base &&
442 bp->b_buffer_length == range_length) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443 /*
Nathan Scottce8e9222006-01-11 15:39:08 +1100444 * If we look at something, bring it to the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445 * front of the list for next time.
446 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100447 atomic_inc(&bp->b_hold);
448 list_move(&bp->b_hash_list, &hash->bh_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449 goto found;
450 }
451 }
452
453 /* No match found */
Nathan Scottce8e9222006-01-11 15:39:08 +1100454 if (new_bp) {
455 _xfs_buf_initialize(new_bp, btp, range_base,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700456 range_length, flags);
Nathan Scottce8e9222006-01-11 15:39:08 +1100457 new_bp->b_hash = hash;
458 list_add(&new_bp->b_hash_list, &hash->bh_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700459 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100460 XFS_STATS_INC(xb_miss_locked);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700461 }
462
463 spin_unlock(&hash->bh_lock);
Nathan Scottce8e9222006-01-11 15:39:08 +1100464 return new_bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465
466found:
467 spin_unlock(&hash->bh_lock);
468
469 /* Attempt to get the semaphore without sleeping,
470 * if this does not work then we need to drop the
471 * spinlock and do a hard attempt on the semaphore.
472 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100473 if (down_trylock(&bp->b_sema)) {
474 if (!(flags & XBF_TRYLOCK)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700475 /* wait for buffer ownership */
Nathan Scottce8e9222006-01-11 15:39:08 +1100476 xfs_buf_lock(bp);
477 XFS_STATS_INC(xb_get_locked_waited);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700478 } else {
479 /* We asked for a trylock and failed, no need
480 * to look at file offset and length here, we
Nathan Scottce8e9222006-01-11 15:39:08 +1100481 * know that this buffer at least overlaps our
482 * buffer and is locked, therefore our buffer
483 * either does not exist, or is this buffer.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700484 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100485 xfs_buf_rele(bp);
486 XFS_STATS_INC(xb_busy_locked);
487 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488 }
489 } else {
490 /* trylock worked */
Nathan Scottce8e9222006-01-11 15:39:08 +1100491 XB_SET_OWNER(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492 }
493
Nathan Scottce8e9222006-01-11 15:39:08 +1100494 if (bp->b_flags & XBF_STALE) {
495 ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
496 bp->b_flags &= XBF_MAPPED;
David Chinner2f926582005-09-05 08:33:35 +1000497 }
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000498
499 trace_xfs_buf_find(bp, flags, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +1100500 XFS_STATS_INC(xb_get_locked);
501 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502}
503
504/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100505 * Assembles a buffer covering the specified range.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700506 * Storage in memory for all portions of the buffer will be allocated,
507 * although backing storage may not be.
508 */
509xfs_buf_t *
Christoph Hellwig6ad112b2009-11-24 18:02:23 +0000510xfs_buf_get(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700511 xfs_buftarg_t *target,/* target for buffer */
Nathan Scott204ab252006-01-11 20:50:22 +1100512 xfs_off_t ioff, /* starting offset of range */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513 size_t isize, /* length of range */
Nathan Scottce8e9222006-01-11 15:39:08 +1100514 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700515{
Nathan Scottce8e9222006-01-11 15:39:08 +1100516 xfs_buf_t *bp, *new_bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700517 int error = 0, i;
518
Nathan Scottce8e9222006-01-11 15:39:08 +1100519 new_bp = xfs_buf_allocate(flags);
520 if (unlikely(!new_bp))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521 return NULL;
522
Nathan Scottce8e9222006-01-11 15:39:08 +1100523 bp = _xfs_buf_find(target, ioff, isize, flags, new_bp);
524 if (bp == new_bp) {
525 error = _xfs_buf_lookup_pages(bp, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526 if (error)
527 goto no_buffer;
528 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100529 xfs_buf_deallocate(new_bp);
530 if (unlikely(bp == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700531 return NULL;
532 }
533
Nathan Scottce8e9222006-01-11 15:39:08 +1100534 for (i = 0; i < bp->b_page_count; i++)
535 mark_page_accessed(bp->b_pages[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700536
Nathan Scottce8e9222006-01-11 15:39:08 +1100537 if (!(bp->b_flags & XBF_MAPPED)) {
538 error = _xfs_buf_map_pages(bp, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700539 if (unlikely(error)) {
540 printk(KERN_WARNING "%s: failed to map pages\n",
Harvey Harrison34a622b2008-04-10 12:19:21 +1000541 __func__);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542 goto no_buffer;
543 }
544 }
545
Nathan Scottce8e9222006-01-11 15:39:08 +1100546 XFS_STATS_INC(xb_get);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700547
548 /*
549 * Always fill in the block number now, the mapped cases can do
550 * their own overlay of this later.
551 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100552 bp->b_bn = ioff;
553 bp->b_count_desired = bp->b_buffer_length;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000555 trace_xfs_buf_get(bp, flags, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +1100556 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700557
558 no_buffer:
Nathan Scottce8e9222006-01-11 15:39:08 +1100559 if (flags & (XBF_LOCK | XBF_TRYLOCK))
560 xfs_buf_unlock(bp);
561 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700562 return NULL;
563}
564
Christoph Hellwig5d765b92008-12-03 12:20:26 +0100565STATIC int
566_xfs_buf_read(
567 xfs_buf_t *bp,
568 xfs_buf_flags_t flags)
569{
570 int status;
571
Christoph Hellwig5d765b92008-12-03 12:20:26 +0100572 ASSERT(!(flags & (XBF_DELWRI|XBF_WRITE)));
573 ASSERT(bp->b_bn != XFS_BUF_DADDR_NULL);
574
575 bp->b_flags &= ~(XBF_WRITE | XBF_ASYNC | XBF_DELWRI | \
576 XBF_READ_AHEAD | _XBF_RUN_QUEUES);
577 bp->b_flags |= flags & (XBF_READ | XBF_ASYNC | \
578 XBF_READ_AHEAD | _XBF_RUN_QUEUES);
579
580 status = xfs_buf_iorequest(bp);
581 if (!status && !(flags & XBF_ASYNC))
582 status = xfs_buf_iowait(bp);
583 return status;
584}
585
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586xfs_buf_t *
Christoph Hellwig6ad112b2009-11-24 18:02:23 +0000587xfs_buf_read(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700588 xfs_buftarg_t *target,
Nathan Scott204ab252006-01-11 20:50:22 +1100589 xfs_off_t ioff,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590 size_t isize,
Nathan Scottce8e9222006-01-11 15:39:08 +1100591 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592{
Nathan Scottce8e9222006-01-11 15:39:08 +1100593 xfs_buf_t *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594
Nathan Scottce8e9222006-01-11 15:39:08 +1100595 flags |= XBF_READ;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700596
Christoph Hellwig6ad112b2009-11-24 18:02:23 +0000597 bp = xfs_buf_get(target, ioff, isize, flags);
Nathan Scottce8e9222006-01-11 15:39:08 +1100598 if (bp) {
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000599 trace_xfs_buf_read(bp, flags, _RET_IP_);
600
Nathan Scottce8e9222006-01-11 15:39:08 +1100601 if (!XFS_BUF_ISDONE(bp)) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100602 XFS_STATS_INC(xb_get_read);
Christoph Hellwig5d765b92008-12-03 12:20:26 +0100603 _xfs_buf_read(bp, flags);
Nathan Scottce8e9222006-01-11 15:39:08 +1100604 } else if (flags & XBF_ASYNC) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605 /*
606 * Read ahead call which is already satisfied,
607 * drop the buffer
608 */
609 goto no_buffer;
610 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611 /* We do not want read in the flags */
Nathan Scottce8e9222006-01-11 15:39:08 +1100612 bp->b_flags &= ~XBF_READ;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613 }
614 }
615
Nathan Scottce8e9222006-01-11 15:39:08 +1100616 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617
618 no_buffer:
Nathan Scottce8e9222006-01-11 15:39:08 +1100619 if (flags & (XBF_LOCK | XBF_TRYLOCK))
620 xfs_buf_unlock(bp);
621 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622 return NULL;
623}
624
625/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100626 * If we are not low on memory then do the readahead in a deadlock
627 * safe manner.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700628 */
629void
Nathan Scottce8e9222006-01-11 15:39:08 +1100630xfs_buf_readahead(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631 xfs_buftarg_t *target,
Nathan Scott204ab252006-01-11 20:50:22 +1100632 xfs_off_t ioff,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700633 size_t isize,
Nathan Scottce8e9222006-01-11 15:39:08 +1100634 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700635{
636 struct backing_dev_info *bdi;
637
Nathan Scottce8e9222006-01-11 15:39:08 +1100638 bdi = target->bt_mapping->backing_dev_info;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700639 if (bdi_read_congested(bdi))
640 return;
641
Nathan Scottce8e9222006-01-11 15:39:08 +1100642 flags |= (XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD);
Christoph Hellwig6ad112b2009-11-24 18:02:23 +0000643 xfs_buf_read(target, ioff, isize, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644}
645
646xfs_buf_t *
Nathan Scottce8e9222006-01-11 15:39:08 +1100647xfs_buf_get_empty(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700648 size_t len,
649 xfs_buftarg_t *target)
650{
Nathan Scottce8e9222006-01-11 15:39:08 +1100651 xfs_buf_t *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700652
Nathan Scottce8e9222006-01-11 15:39:08 +1100653 bp = xfs_buf_allocate(0);
654 if (bp)
655 _xfs_buf_initialize(bp, target, 0, len, 0);
656 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657}
658
659static inline struct page *
660mem_to_page(
661 void *addr)
662{
Christoph Lameter9e2779f2008-02-04 22:28:34 -0800663 if ((!is_vmalloc_addr(addr))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700664 return virt_to_page(addr);
665 } else {
666 return vmalloc_to_page(addr);
667 }
668}
669
670int
Nathan Scottce8e9222006-01-11 15:39:08 +1100671xfs_buf_associate_memory(
672 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700673 void *mem,
674 size_t len)
675{
676 int rval;
677 int i = 0;
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100678 unsigned long pageaddr;
679 unsigned long offset;
680 size_t buflen;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700681 int page_count;
682
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100683 pageaddr = (unsigned long)mem & PAGE_CACHE_MASK;
684 offset = (unsigned long)mem - pageaddr;
685 buflen = PAGE_CACHE_ALIGN(len + offset);
686 page_count = buflen >> PAGE_CACHE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700687
688 /* Free any previous set of page pointers */
Nathan Scottce8e9222006-01-11 15:39:08 +1100689 if (bp->b_pages)
690 _xfs_buf_free_pages(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700691
Nathan Scottce8e9222006-01-11 15:39:08 +1100692 bp->b_pages = NULL;
693 bp->b_addr = mem;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694
Christoph Hellwig36fae172009-07-18 18:14:58 -0400695 rval = _xfs_buf_get_pages(bp, page_count, XBF_DONT_BLOCK);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700696 if (rval)
697 return rval;
698
Nathan Scottce8e9222006-01-11 15:39:08 +1100699 bp->b_offset = offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700700
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100701 for (i = 0; i < bp->b_page_count; i++) {
702 bp->b_pages[i] = mem_to_page((void *)pageaddr);
703 pageaddr += PAGE_CACHE_SIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700704 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100706 bp->b_count_desired = len;
707 bp->b_buffer_length = buflen;
Nathan Scottce8e9222006-01-11 15:39:08 +1100708 bp->b_flags |= XBF_MAPPED;
Christoph Hellwig6ab455e2008-05-19 16:34:42 +1000709 bp->b_flags &= ~_XBF_PAGE_LOCKED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700710
711 return 0;
712}
713
714xfs_buf_t *
Nathan Scottce8e9222006-01-11 15:39:08 +1100715xfs_buf_get_noaddr(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700716 size_t len,
717 xfs_buftarg_t *target)
718{
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000719 unsigned long page_count = PAGE_ALIGN(len) >> PAGE_SHIFT;
720 int error, i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700721 xfs_buf_t *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700722
Nathan Scottce8e9222006-01-11 15:39:08 +1100723 bp = xfs_buf_allocate(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700724 if (unlikely(bp == NULL))
725 goto fail;
Nathan Scottce8e9222006-01-11 15:39:08 +1100726 _xfs_buf_initialize(bp, target, 0, len, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700727
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000728 error = _xfs_buf_get_pages(bp, page_count, 0);
729 if (error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700730 goto fail_free_buf;
731
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000732 for (i = 0; i < page_count; i++) {
733 bp->b_pages[i] = alloc_page(GFP_KERNEL);
734 if (!bp->b_pages[i])
735 goto fail_free_mem;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736 }
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000737 bp->b_flags |= _XBF_PAGES;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700738
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000739 error = _xfs_buf_map_pages(bp, XBF_MAPPED);
740 if (unlikely(error)) {
741 printk(KERN_WARNING "%s: failed to map pages\n",
Harvey Harrison34a622b2008-04-10 12:19:21 +1000742 __func__);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700743 goto fail_free_mem;
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000744 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700745
Nathan Scottce8e9222006-01-11 15:39:08 +1100746 xfs_buf_unlock(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700747
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000748 trace_xfs_buf_get_noaddr(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700749 return bp;
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000750
Linus Torvalds1da177e2005-04-16 15:20:36 -0700751 fail_free_mem:
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000752 while (--i >= 0)
753 __free_page(bp->b_pages[i]);
Christoph Hellwigca165b82007-05-24 15:21:11 +1000754 _xfs_buf_free_pages(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700755 fail_free_buf:
Christoph Hellwigca165b82007-05-24 15:21:11 +1000756 xfs_buf_deallocate(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700757 fail:
758 return NULL;
759}
760
761/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700762 * Increment reference count on buffer, to hold the buffer concurrently
763 * with another thread which may release (free) the buffer asynchronously.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700764 * Must hold the buffer already to call this function.
765 */
766void
Nathan Scottce8e9222006-01-11 15:39:08 +1100767xfs_buf_hold(
768 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700769{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000770 trace_xfs_buf_hold(bp, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +1100771 atomic_inc(&bp->b_hold);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700772}
773
774/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100775 * Releases a hold on the specified buffer. If the
776 * the hold count is 1, calls xfs_buf_free.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700777 */
778void
Nathan Scottce8e9222006-01-11 15:39:08 +1100779xfs_buf_rele(
780 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700781{
Nathan Scottce8e9222006-01-11 15:39:08 +1100782 xfs_bufhash_t *hash = bp->b_hash;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700783
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000784 trace_xfs_buf_rele(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700785
Nathan Scottfad3aa12006-02-01 12:14:52 +1100786 if (unlikely(!hash)) {
787 ASSERT(!bp->b_relse);
788 if (atomic_dec_and_test(&bp->b_hold))
789 xfs_buf_free(bp);
790 return;
791 }
792
Lachlan McIlroy37906892008-08-13 15:42:10 +1000793 ASSERT(atomic_read(&bp->b_hold) > 0);
Nathan Scottce8e9222006-01-11 15:39:08 +1100794 if (atomic_dec_and_lock(&bp->b_hold, &hash->bh_lock)) {
795 if (bp->b_relse) {
796 atomic_inc(&bp->b_hold);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797 spin_unlock(&hash->bh_lock);
Nathan Scottce8e9222006-01-11 15:39:08 +1100798 (*(bp->b_relse)) (bp);
799 } else if (bp->b_flags & XBF_FS_MANAGED) {
Christoph Hellwig7f14d0a2005-11-02 15:09:35 +1100800 spin_unlock(&hash->bh_lock);
801 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100802 ASSERT(!(bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)));
803 list_del_init(&bp->b_hash_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700804 spin_unlock(&hash->bh_lock);
Nathan Scottce8e9222006-01-11 15:39:08 +1100805 xfs_buf_free(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700806 }
807 }
808}
809
810
811/*
812 * Mutual exclusion on buffers. Locking model:
813 *
814 * Buffers associated with inodes for which buffer locking
815 * is not enabled are not protected by semaphores, and are
816 * assumed to be exclusively owned by the caller. There is a
817 * spinlock in the buffer, used by the caller when concurrent
818 * access is possible.
819 */
820
821/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100822 * Locks a buffer object, if it is not already locked.
823 * Note that this in no way locks the underlying pages, so it is only
824 * useful for synchronizing concurrent use of buffer objects, not for
825 * synchronizing independent access to the underlying pages.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700826 */
827int
Nathan Scottce8e9222006-01-11 15:39:08 +1100828xfs_buf_cond_lock(
829 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700830{
831 int locked;
832
Nathan Scottce8e9222006-01-11 15:39:08 +1100833 locked = down_trylock(&bp->b_sema) == 0;
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000834 if (locked)
Nathan Scottce8e9222006-01-11 15:39:08 +1100835 XB_SET_OWNER(bp);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000836
837 trace_xfs_buf_cond_lock(bp, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +1100838 return locked ? 0 : -EBUSY;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700839}
840
Linus Torvalds1da177e2005-04-16 15:20:36 -0700841int
Nathan Scottce8e9222006-01-11 15:39:08 +1100842xfs_buf_lock_value(
843 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700844{
Stephen Rothwelladaa6932008-04-22 15:26:13 +1000845 return bp->b_sema.count;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700846}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700847
848/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100849 * Locks a buffer object.
850 * Note that this in no way locks the underlying pages, so it is only
851 * useful for synchronizing concurrent use of buffer objects, not for
852 * synchronizing independent access to the underlying pages.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700853 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100854void
855xfs_buf_lock(
856 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700857{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000858 trace_xfs_buf_lock(bp, _RET_IP_);
859
Nathan Scottce8e9222006-01-11 15:39:08 +1100860 if (atomic_read(&bp->b_io_remaining))
861 blk_run_address_space(bp->b_target->bt_mapping);
862 down(&bp->b_sema);
863 XB_SET_OWNER(bp);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000864
865 trace_xfs_buf_lock_done(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700866}
867
868/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100869 * Releases the lock on the buffer object.
David Chinner2f926582005-09-05 08:33:35 +1000870 * If the buffer is marked delwri but is not queued, do so before we
Nathan Scottce8e9222006-01-11 15:39:08 +1100871 * unlock the buffer as we need to set flags correctly. We also need to
David Chinner2f926582005-09-05 08:33:35 +1000872 * take a reference for the delwri queue because the unlocker is going to
873 * drop their's and they don't know we just queued it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700874 */
875void
Nathan Scottce8e9222006-01-11 15:39:08 +1100876xfs_buf_unlock(
877 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700878{
Nathan Scottce8e9222006-01-11 15:39:08 +1100879 if ((bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)) == XBF_DELWRI) {
880 atomic_inc(&bp->b_hold);
881 bp->b_flags |= XBF_ASYNC;
882 xfs_buf_delwri_queue(bp, 0);
David Chinner2f926582005-09-05 08:33:35 +1000883 }
884
Nathan Scottce8e9222006-01-11 15:39:08 +1100885 XB_CLEAR_OWNER(bp);
886 up(&bp->b_sema);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000887
888 trace_xfs_buf_unlock(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889}
890
891
892/*
893 * Pinning Buffer Storage in Memory
Nathan Scottce8e9222006-01-11 15:39:08 +1100894 * Ensure that no attempt to force a buffer to disk will succeed.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700895 */
896void
Nathan Scottce8e9222006-01-11 15:39:08 +1100897xfs_buf_pin(
898 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700899{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000900 trace_xfs_buf_pin(bp, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +1100901 atomic_inc(&bp->b_pin_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700902}
903
Linus Torvalds1da177e2005-04-16 15:20:36 -0700904void
Nathan Scottce8e9222006-01-11 15:39:08 +1100905xfs_buf_unpin(
906 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700907{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000908 trace_xfs_buf_unpin(bp, _RET_IP_);
909
Nathan Scottce8e9222006-01-11 15:39:08 +1100910 if (atomic_dec_and_test(&bp->b_pin_count))
911 wake_up_all(&bp->b_waiters);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700912}
913
914int
Nathan Scottce8e9222006-01-11 15:39:08 +1100915xfs_buf_ispin(
916 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700917{
Nathan Scottce8e9222006-01-11 15:39:08 +1100918 return atomic_read(&bp->b_pin_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700919}
920
Nathan Scottce8e9222006-01-11 15:39:08 +1100921STATIC void
922xfs_buf_wait_unpin(
923 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700924{
925 DECLARE_WAITQUEUE (wait, current);
926
Nathan Scottce8e9222006-01-11 15:39:08 +1100927 if (atomic_read(&bp->b_pin_count) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700928 return;
929
Nathan Scottce8e9222006-01-11 15:39:08 +1100930 add_wait_queue(&bp->b_waiters, &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700931 for (;;) {
932 set_current_state(TASK_UNINTERRUPTIBLE);
Nathan Scottce8e9222006-01-11 15:39:08 +1100933 if (atomic_read(&bp->b_pin_count) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700934 break;
Nathan Scottce8e9222006-01-11 15:39:08 +1100935 if (atomic_read(&bp->b_io_remaining))
936 blk_run_address_space(bp->b_target->bt_mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700937 schedule();
938 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100939 remove_wait_queue(&bp->b_waiters, &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700940 set_current_state(TASK_RUNNING);
941}
942
943/*
944 * Buffer Utility Routines
945 */
946
Linus Torvalds1da177e2005-04-16 15:20:36 -0700947STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +1100948xfs_buf_iodone_work(
David Howellsc4028952006-11-22 14:57:56 +0000949 struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950{
David Howellsc4028952006-11-22 14:57:56 +0000951 xfs_buf_t *bp =
952 container_of(work, xfs_buf_t, b_iodone_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700953
David Chinner0bfefc42007-05-14 18:24:23 +1000954 /*
955 * We can get an EOPNOTSUPP to ordered writes. Here we clear the
956 * ordered flag and reissue them. Because we can't tell the higher
957 * layers directly that they should not issue ordered I/O anymore, they
Christoph Hellwig73f6aa42008-10-10 17:28:29 +1100958 * need to check if the _XFS_BARRIER_FAILED flag was set during I/O completion.
David Chinner0bfefc42007-05-14 18:24:23 +1000959 */
960 if ((bp->b_error == EOPNOTSUPP) &&
961 (bp->b_flags & (XBF_ORDERED|XBF_ASYNC)) == (XBF_ORDERED|XBF_ASYNC)) {
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000962 trace_xfs_buf_ordered_retry(bp, _RET_IP_);
David Chinner0bfefc42007-05-14 18:24:23 +1000963 bp->b_flags &= ~XBF_ORDERED;
Christoph Hellwig73f6aa42008-10-10 17:28:29 +1100964 bp->b_flags |= _XFS_BARRIER_FAILED;
David Chinner0bfefc42007-05-14 18:24:23 +1000965 xfs_buf_iorequest(bp);
966 } else if (bp->b_iodone)
Nathan Scottce8e9222006-01-11 15:39:08 +1100967 (*(bp->b_iodone))(bp);
968 else if (bp->b_flags & XBF_ASYNC)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969 xfs_buf_relse(bp);
970}
971
972void
Nathan Scottce8e9222006-01-11 15:39:08 +1100973xfs_buf_ioend(
974 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700975 int schedule)
976{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000977 trace_xfs_buf_iodone(bp, _RET_IP_);
978
Lachlan McIlroy77be55a2007-11-23 16:31:00 +1100979 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
Nathan Scottce8e9222006-01-11 15:39:08 +1100980 if (bp->b_error == 0)
981 bp->b_flags |= XBF_DONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982
Nathan Scottce8e9222006-01-11 15:39:08 +1100983 if ((bp->b_iodone) || (bp->b_flags & XBF_ASYNC)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700984 if (schedule) {
David Howellsc4028952006-11-22 14:57:56 +0000985 INIT_WORK(&bp->b_iodone_work, xfs_buf_iodone_work);
Nathan Scottce8e9222006-01-11 15:39:08 +1100986 queue_work(xfslogd_workqueue, &bp->b_iodone_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700987 } else {
David Howellsc4028952006-11-22 14:57:56 +0000988 xfs_buf_iodone_work(&bp->b_iodone_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700989 }
990 } else {
David Chinnerb4dd3302008-08-13 16:36:11 +1000991 complete(&bp->b_iowait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700992 }
993}
994
Linus Torvalds1da177e2005-04-16 15:20:36 -0700995void
Nathan Scottce8e9222006-01-11 15:39:08 +1100996xfs_buf_ioerror(
997 xfs_buf_t *bp,
998 int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700999{
1000 ASSERT(error >= 0 && error <= 0xffff);
Nathan Scottce8e9222006-01-11 15:39:08 +11001001 bp->b_error = (unsigned short)error;
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001002 trace_xfs_buf_ioerror(bp, error, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001003}
1004
Linus Torvalds1da177e2005-04-16 15:20:36 -07001005int
Christoph Hellwig64e0bc72010-01-13 22:17:58 +00001006xfs_bwrite(
1007 struct xfs_mount *mp,
Christoph Hellwig5d765b92008-12-03 12:20:26 +01001008 struct xfs_buf *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001009{
Christoph Hellwig64e0bc72010-01-13 22:17:58 +00001010 int iowait = (bp->b_flags & XBF_ASYNC) == 0;
1011 int error = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001012
Christoph Hellwig64e0bc72010-01-13 22:17:58 +00001013 bp->b_strat = xfs_bdstrat_cb;
1014 bp->b_mount = mp;
1015 bp->b_flags |= XBF_WRITE;
1016 if (!iowait)
1017 bp->b_flags |= _XBF_RUN_QUEUES;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001018
Christoph Hellwig5d765b92008-12-03 12:20:26 +01001019 xfs_buf_delwri_dequeue(bp);
Christoph Hellwig64e0bc72010-01-13 22:17:58 +00001020 xfs_buf_iostrategy(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001021
Christoph Hellwig64e0bc72010-01-13 22:17:58 +00001022 if (iowait) {
1023 error = xfs_buf_iowait(bp);
1024 if (error)
1025 xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
1026 xfs_buf_relse(bp);
1027 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001028
Christoph Hellwig64e0bc72010-01-13 22:17:58 +00001029 return error;
Christoph Hellwig5d765b92008-12-03 12:20:26 +01001030}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031
Christoph Hellwig5d765b92008-12-03 12:20:26 +01001032void
1033xfs_bdwrite(
1034 void *mp,
1035 struct xfs_buf *bp)
1036{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001037 trace_xfs_buf_bdwrite(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001038
Christoph Hellwig5d765b92008-12-03 12:20:26 +01001039 bp->b_strat = xfs_bdstrat_cb;
Christoph Hellwig15ac08a2008-12-09 04:47:30 -05001040 bp->b_mount = mp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001041
Christoph Hellwig5d765b92008-12-03 12:20:26 +01001042 bp->b_flags &= ~XBF_READ;
1043 bp->b_flags |= (XBF_DELWRI | XBF_ASYNC);
1044
1045 xfs_buf_delwri_queue(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001046}
1047
Christoph Hellwig4e234712010-01-13 22:17:56 +00001048/*
1049 * Called when we want to stop a buffer from getting written or read.
1050 * We attach the EIO error, muck with its flags, and call biodone
1051 * so that the proper iodone callbacks get called.
1052 */
1053STATIC int
1054xfs_bioerror(
1055 xfs_buf_t *bp)
1056{
1057#ifdef XFSERRORDEBUG
1058 ASSERT(XFS_BUF_ISREAD(bp) || bp->b_iodone);
1059#endif
1060
1061 /*
1062 * No need to wait until the buffer is unpinned, we aren't flushing it.
1063 */
1064 XFS_BUF_ERROR(bp, EIO);
1065
1066 /*
1067 * We're calling biodone, so delete XBF_DONE flag.
1068 */
1069 XFS_BUF_UNREAD(bp);
1070 XFS_BUF_UNDELAYWRITE(bp);
1071 XFS_BUF_UNDONE(bp);
1072 XFS_BUF_STALE(bp);
1073
1074 XFS_BUF_CLR_BDSTRAT_FUNC(bp);
1075 xfs_biodone(bp);
1076
1077 return EIO;
1078}
1079
1080/*
1081 * Same as xfs_bioerror, except that we are releasing the buffer
1082 * here ourselves, and avoiding the biodone call.
1083 * This is meant for userdata errors; metadata bufs come with
1084 * iodone functions attached, so that we can track down errors.
1085 */
1086STATIC int
1087xfs_bioerror_relse(
1088 struct xfs_buf *bp)
1089{
1090 int64_t fl = XFS_BUF_BFLAGS(bp);
1091 /*
1092 * No need to wait until the buffer is unpinned.
1093 * We aren't flushing it.
1094 *
1095 * chunkhold expects B_DONE to be set, whether
1096 * we actually finish the I/O or not. We don't want to
1097 * change that interface.
1098 */
1099 XFS_BUF_UNREAD(bp);
1100 XFS_BUF_UNDELAYWRITE(bp);
1101 XFS_BUF_DONE(bp);
1102 XFS_BUF_STALE(bp);
1103 XFS_BUF_CLR_IODONE_FUNC(bp);
1104 XFS_BUF_CLR_BDSTRAT_FUNC(bp);
Christoph Hellwig0cadda12010-01-19 09:56:44 +00001105 if (!(fl & XBF_ASYNC)) {
Christoph Hellwig4e234712010-01-13 22:17:56 +00001106 /*
1107 * Mark b_error and B_ERROR _both_.
1108 * Lot's of chunkcache code assumes that.
1109 * There's no reason to mark error for
1110 * ASYNC buffers.
1111 */
1112 XFS_BUF_ERROR(bp, EIO);
1113 XFS_BUF_FINISH_IOWAIT(bp);
1114 } else {
1115 xfs_buf_relse(bp);
1116 }
1117
1118 return EIO;
1119}
1120
1121
1122/*
1123 * All xfs metadata buffers except log state machine buffers
1124 * get this attached as their b_bdstrat callback function.
1125 * This is so that we can catch a buffer
1126 * after prematurely unpinning it to forcibly shutdown the filesystem.
1127 */
1128int
1129xfs_bdstrat_cb(
1130 struct xfs_buf *bp)
1131{
1132 if (XFS_FORCED_SHUTDOWN(bp->b_mount)) {
1133 trace_xfs_bdstrat_shut(bp, _RET_IP_);
1134 /*
1135 * Metadata write that didn't get logged but
1136 * written delayed anyway. These aren't associated
1137 * with a transaction, and can be ignored.
1138 */
1139 if (!bp->b_iodone && !XFS_BUF_ISREAD(bp))
1140 return xfs_bioerror_relse(bp);
1141 else
1142 return xfs_bioerror(bp);
1143 }
1144
1145 xfs_buf_iorequest(bp);
1146 return 0;
1147}
1148
1149/*
1150 * Wrapper around bdstrat so that we can stop data from going to disk in case
1151 * we are shutting down the filesystem. Typically user data goes thru this
1152 * path; one of the exceptions is the superblock.
1153 */
1154void
1155xfsbdstrat(
1156 struct xfs_mount *mp,
1157 struct xfs_buf *bp)
1158{
1159 if (XFS_FORCED_SHUTDOWN(mp)) {
1160 trace_xfs_bdstrat_shut(bp, _RET_IP_);
1161 xfs_bioerror_relse(bp);
1162 return;
1163 }
1164
1165 xfs_buf_iorequest(bp);
1166}
1167
Christoph Hellwigb8f82a42009-11-14 16:17:22 +00001168STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001169_xfs_buf_ioend(
1170 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001171 int schedule)
1172{
Christoph Hellwig6ab455e2008-05-19 16:34:42 +10001173 if (atomic_dec_and_test(&bp->b_io_remaining) == 1) {
1174 bp->b_flags &= ~_XBF_PAGE_LOCKED;
Nathan Scottce8e9222006-01-11 15:39:08 +11001175 xfs_buf_ioend(bp, schedule);
Christoph Hellwig6ab455e2008-05-19 16:34:42 +10001176 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001177}
1178
Al Viro782e3b32007-10-12 07:17:47 +01001179STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001180xfs_buf_bio_end_io(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001181 struct bio *bio,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001182 int error)
1183{
Nathan Scottce8e9222006-01-11 15:39:08 +11001184 xfs_buf_t *bp = (xfs_buf_t *)bio->bi_private;
1185 unsigned int blocksize = bp->b_target->bt_bsize;
Nathan Scotteedb5532005-09-02 16:39:56 +10001186 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001187
Lachlan McIlroycfbe5262008-12-12 15:27:25 +11001188 xfs_buf_ioerror(bp, -error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001189
James Bottomley73c77e22010-01-25 11:42:24 -06001190 if (!error && xfs_buf_is_vmapped(bp) && (bp->b_flags & XBF_READ))
1191 invalidate_kernel_vmap_range(bp->b_addr, xfs_buf_vmap_len(bp));
1192
Nathan Scotteedb5532005-09-02 16:39:56 +10001193 do {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001194 struct page *page = bvec->bv_page;
1195
Nathan Scott948ecdb2006-09-28 11:03:13 +10001196 ASSERT(!PagePrivate(page));
Nathan Scottce8e9222006-01-11 15:39:08 +11001197 if (unlikely(bp->b_error)) {
1198 if (bp->b_flags & XBF_READ)
Nathan Scotteedb5532005-09-02 16:39:56 +10001199 ClearPageUptodate(page);
Nathan Scottce8e9222006-01-11 15:39:08 +11001200 } else if (blocksize >= PAGE_CACHE_SIZE) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001201 SetPageUptodate(page);
1202 } else if (!PagePrivate(page) &&
Nathan Scottce8e9222006-01-11 15:39:08 +11001203 (bp->b_flags & _XBF_PAGE_CACHE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001204 set_page_region(page, bvec->bv_offset, bvec->bv_len);
1205 }
1206
Nathan Scotteedb5532005-09-02 16:39:56 +10001207 if (--bvec >= bio->bi_io_vec)
1208 prefetchw(&bvec->bv_page->flags);
Christoph Hellwig6ab455e2008-05-19 16:34:42 +10001209
1210 if (bp->b_flags & _XBF_PAGE_LOCKED)
1211 unlock_page(page);
Nathan Scotteedb5532005-09-02 16:39:56 +10001212 } while (bvec >= bio->bi_io_vec);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001213
Nathan Scottce8e9222006-01-11 15:39:08 +11001214 _xfs_buf_ioend(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001215 bio_put(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001216}
1217
1218STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001219_xfs_buf_ioapply(
1220 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001221{
Christoph Hellwiga9759f22007-12-07 14:07:08 +11001222 int rw, map_i, total_nr_pages, nr_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001223 struct bio *bio;
Nathan Scottce8e9222006-01-11 15:39:08 +11001224 int offset = bp->b_offset;
1225 int size = bp->b_count_desired;
1226 sector_t sector = bp->b_bn;
1227 unsigned int blocksize = bp->b_target->bt_bsize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001228
Nathan Scottce8e9222006-01-11 15:39:08 +11001229 total_nr_pages = bp->b_page_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001230 map_i = 0;
1231
Nathan Scottce8e9222006-01-11 15:39:08 +11001232 if (bp->b_flags & XBF_ORDERED) {
1233 ASSERT(!(bp->b_flags & XBF_READ));
Christoph Hellwigf538d4d2005-11-02 10:26:59 +11001234 rw = WRITE_BARRIER;
Dave Chinner2ee1aba2009-11-24 18:03:15 +00001235 } else if (bp->b_flags & XBF_LOG_BUFFER) {
Nathan Scott51bdd702006-09-28 11:01:57 +10001236 ASSERT(!(bp->b_flags & XBF_READ_AHEAD));
1237 bp->b_flags &= ~_XBF_RUN_QUEUES;
1238 rw = (bp->b_flags & XBF_WRITE) ? WRITE_SYNC : READ_SYNC;
Dave Chinner2ee1aba2009-11-24 18:03:15 +00001239 } else if (bp->b_flags & _XBF_RUN_QUEUES) {
1240 ASSERT(!(bp->b_flags & XBF_READ_AHEAD));
1241 bp->b_flags &= ~_XBF_RUN_QUEUES;
1242 rw = (bp->b_flags & XBF_WRITE) ? WRITE_META : READ_META;
Nathan Scott51bdd702006-09-28 11:01:57 +10001243 } else {
1244 rw = (bp->b_flags & XBF_WRITE) ? WRITE :
1245 (bp->b_flags & XBF_READ_AHEAD) ? READA : READ;
Christoph Hellwigf538d4d2005-11-02 10:26:59 +11001246 }
1247
Nathan Scottce8e9222006-01-11 15:39:08 +11001248 /* Special code path for reading a sub page size buffer in --
Linus Torvalds1da177e2005-04-16 15:20:36 -07001249 * we populate up the whole page, and hence the other metadata
1250 * in the same page. This optimization is only valid when the
Nathan Scottce8e9222006-01-11 15:39:08 +11001251 * filesystem block size is not smaller than the page size.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001252 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001253 if ((bp->b_buffer_length < PAGE_CACHE_SIZE) &&
Christoph Hellwig6ab455e2008-05-19 16:34:42 +10001254 ((bp->b_flags & (XBF_READ|_XBF_PAGE_LOCKED)) ==
1255 (XBF_READ|_XBF_PAGE_LOCKED)) &&
Nathan Scottce8e9222006-01-11 15:39:08 +11001256 (blocksize >= PAGE_CACHE_SIZE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001257 bio = bio_alloc(GFP_NOIO, 1);
1258
Nathan Scottce8e9222006-01-11 15:39:08 +11001259 bio->bi_bdev = bp->b_target->bt_bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001260 bio->bi_sector = sector - (offset >> BBSHIFT);
Nathan Scottce8e9222006-01-11 15:39:08 +11001261 bio->bi_end_io = xfs_buf_bio_end_io;
1262 bio->bi_private = bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001263
Nathan Scottce8e9222006-01-11 15:39:08 +11001264 bio_add_page(bio, bp->b_pages[0], PAGE_CACHE_SIZE, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001265 size = 0;
1266
Nathan Scottce8e9222006-01-11 15:39:08 +11001267 atomic_inc(&bp->b_io_remaining);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001268
1269 goto submit_io;
1270 }
1271
Linus Torvalds1da177e2005-04-16 15:20:36 -07001272next_chunk:
Nathan Scottce8e9222006-01-11 15:39:08 +11001273 atomic_inc(&bp->b_io_remaining);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001274 nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1275 if (nr_pages > total_nr_pages)
1276 nr_pages = total_nr_pages;
1277
1278 bio = bio_alloc(GFP_NOIO, nr_pages);
Nathan Scottce8e9222006-01-11 15:39:08 +11001279 bio->bi_bdev = bp->b_target->bt_bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001280 bio->bi_sector = sector;
Nathan Scottce8e9222006-01-11 15:39:08 +11001281 bio->bi_end_io = xfs_buf_bio_end_io;
1282 bio->bi_private = bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001283
1284 for (; size && nr_pages; nr_pages--, map_i++) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001285 int rbytes, nbytes = PAGE_CACHE_SIZE - offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001286
1287 if (nbytes > size)
1288 nbytes = size;
1289
Nathan Scottce8e9222006-01-11 15:39:08 +11001290 rbytes = bio_add_page(bio, bp->b_pages[map_i], nbytes, offset);
1291 if (rbytes < nbytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001292 break;
1293
1294 offset = 0;
1295 sector += nbytes >> BBSHIFT;
1296 size -= nbytes;
1297 total_nr_pages--;
1298 }
1299
1300submit_io:
1301 if (likely(bio->bi_size)) {
James Bottomley73c77e22010-01-25 11:42:24 -06001302 if (xfs_buf_is_vmapped(bp)) {
1303 flush_kernel_vmap_range(bp->b_addr,
1304 xfs_buf_vmap_len(bp));
1305 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001306 submit_bio(rw, bio);
1307 if (size)
1308 goto next_chunk;
1309 } else {
1310 bio_put(bio);
Nathan Scottce8e9222006-01-11 15:39:08 +11001311 xfs_buf_ioerror(bp, EIO);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001312 }
1313}
1314
Linus Torvalds1da177e2005-04-16 15:20:36 -07001315int
Nathan Scottce8e9222006-01-11 15:39:08 +11001316xfs_buf_iorequest(
1317 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001318{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001319 trace_xfs_buf_iorequest(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001320
Nathan Scottce8e9222006-01-11 15:39:08 +11001321 if (bp->b_flags & XBF_DELWRI) {
1322 xfs_buf_delwri_queue(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001323 return 0;
1324 }
1325
Nathan Scottce8e9222006-01-11 15:39:08 +11001326 if (bp->b_flags & XBF_WRITE) {
1327 xfs_buf_wait_unpin(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001328 }
1329
Nathan Scottce8e9222006-01-11 15:39:08 +11001330 xfs_buf_hold(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001331
1332 /* Set the count to 1 initially, this will stop an I/O
1333 * completion callout which happens before we have started
Nathan Scottce8e9222006-01-11 15:39:08 +11001334 * all the I/O from calling xfs_buf_ioend too early.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001335 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001336 atomic_set(&bp->b_io_remaining, 1);
1337 _xfs_buf_ioapply(bp);
1338 _xfs_buf_ioend(bp, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001339
Nathan Scottce8e9222006-01-11 15:39:08 +11001340 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001341 return 0;
1342}
1343
1344/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001345 * Waits for I/O to complete on the buffer supplied.
1346 * It returns immediately if no I/O is pending.
1347 * It returns the I/O error code, if any, or 0 if there was no error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001348 */
1349int
Nathan Scottce8e9222006-01-11 15:39:08 +11001350xfs_buf_iowait(
1351 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001352{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001353 trace_xfs_buf_iowait(bp, _RET_IP_);
1354
Nathan Scottce8e9222006-01-11 15:39:08 +11001355 if (atomic_read(&bp->b_io_remaining))
1356 blk_run_address_space(bp->b_target->bt_mapping);
David Chinnerb4dd3302008-08-13 16:36:11 +10001357 wait_for_completion(&bp->b_iowait);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001358
1359 trace_xfs_buf_iowait_done(bp, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +11001360 return bp->b_error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001361}
1362
Nathan Scottce8e9222006-01-11 15:39:08 +11001363xfs_caddr_t
1364xfs_buf_offset(
1365 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001366 size_t offset)
1367{
1368 struct page *page;
1369
Nathan Scottce8e9222006-01-11 15:39:08 +11001370 if (bp->b_flags & XBF_MAPPED)
1371 return XFS_BUF_PTR(bp) + offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001372
Nathan Scottce8e9222006-01-11 15:39:08 +11001373 offset += bp->b_offset;
1374 page = bp->b_pages[offset >> PAGE_CACHE_SHIFT];
1375 return (xfs_caddr_t)page_address(page) + (offset & (PAGE_CACHE_SIZE-1));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001376}
1377
1378/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001379 * Move data into or out of a buffer.
1380 */
1381void
Nathan Scottce8e9222006-01-11 15:39:08 +11001382xfs_buf_iomove(
1383 xfs_buf_t *bp, /* buffer to process */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001384 size_t boff, /* starting buffer offset */
1385 size_t bsize, /* length to copy */
Dave Chinnerb9c48642010-01-20 10:47:39 +11001386 void *data, /* data address */
Nathan Scottce8e9222006-01-11 15:39:08 +11001387 xfs_buf_rw_t mode) /* read/write/zero flag */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001388{
1389 size_t bend, cpoff, csize;
1390 struct page *page;
1391
1392 bend = boff + bsize;
1393 while (boff < bend) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001394 page = bp->b_pages[xfs_buf_btoct(boff + bp->b_offset)];
1395 cpoff = xfs_buf_poff(boff + bp->b_offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001396 csize = min_t(size_t,
Nathan Scottce8e9222006-01-11 15:39:08 +11001397 PAGE_CACHE_SIZE-cpoff, bp->b_count_desired-boff);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001398
1399 ASSERT(((csize + cpoff) <= PAGE_CACHE_SIZE));
1400
1401 switch (mode) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001402 case XBRW_ZERO:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001403 memset(page_address(page) + cpoff, 0, csize);
1404 break;
Nathan Scottce8e9222006-01-11 15:39:08 +11001405 case XBRW_READ:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001406 memcpy(data, page_address(page) + cpoff, csize);
1407 break;
Nathan Scottce8e9222006-01-11 15:39:08 +11001408 case XBRW_WRITE:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001409 memcpy(page_address(page) + cpoff, data, csize);
1410 }
1411
1412 boff += csize;
1413 data += csize;
1414 }
1415}
1416
1417/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001418 * Handling of buffer targets (buftargs).
Linus Torvalds1da177e2005-04-16 15:20:36 -07001419 */
1420
1421/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001422 * Wait for any bufs with callbacks that have been submitted but
1423 * have not yet returned... walk the hash list for the target.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001424 */
1425void
1426xfs_wait_buftarg(
1427 xfs_buftarg_t *btp)
1428{
1429 xfs_buf_t *bp, *n;
1430 xfs_bufhash_t *hash;
1431 uint i;
1432
1433 for (i = 0; i < (1 << btp->bt_hashshift); i++) {
1434 hash = &btp->bt_hash[i];
1435again:
1436 spin_lock(&hash->bh_lock);
Nathan Scottce8e9222006-01-11 15:39:08 +11001437 list_for_each_entry_safe(bp, n, &hash->bh_list, b_hash_list) {
1438 ASSERT(btp == bp->b_target);
1439 if (!(bp->b_flags & XBF_FS_MANAGED)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001440 spin_unlock(&hash->bh_lock);
David Chinner2f926582005-09-05 08:33:35 +10001441 /*
1442 * Catch superblock reference count leaks
1443 * immediately
1444 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001445 BUG_ON(bp->b_bn == 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001446 delay(100);
1447 goto again;
1448 }
1449 }
1450 spin_unlock(&hash->bh_lock);
1451 }
1452}
1453
1454/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001455 * Allocate buffer hash table for a given target.
1456 * For devices containing metadata (i.e. not the log/realtime devices)
1457 * we need to allocate a much larger hash table.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001458 */
1459STATIC void
1460xfs_alloc_bufhash(
1461 xfs_buftarg_t *btp,
1462 int external)
1463{
1464 unsigned int i;
1465
1466 btp->bt_hashshift = external ? 3 : 8; /* 8 or 256 buckets */
1467 btp->bt_hashmask = (1 << btp->bt_hashshift) - 1;
Christoph Hellwigbdfb0432010-01-20 21:55:30 +00001468 btp->bt_hash = kmem_zalloc_large((1 << btp->bt_hashshift) *
1469 sizeof(xfs_bufhash_t));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001470 for (i = 0; i < (1 << btp->bt_hashshift); i++) {
1471 spin_lock_init(&btp->bt_hash[i].bh_lock);
1472 INIT_LIST_HEAD(&btp->bt_hash[i].bh_list);
1473 }
1474}
1475
1476STATIC void
1477xfs_free_bufhash(
1478 xfs_buftarg_t *btp)
1479{
Christoph Hellwigbdfb0432010-01-20 21:55:30 +00001480 kmem_free_large(btp->bt_hash);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001481 btp->bt_hash = NULL;
1482}
1483
David Chinnera6867a62006-01-11 15:37:58 +11001484/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001485 * buftarg list for delwrite queue processing
David Chinnera6867a62006-01-11 15:37:58 +11001486 */
Tim Shimmine6a0e9c2007-05-08 13:49:59 +10001487static LIST_HEAD(xfs_buftarg_list);
David Chinner7989cb82007-02-10 18:34:56 +11001488static DEFINE_SPINLOCK(xfs_buftarg_lock);
David Chinnera6867a62006-01-11 15:37:58 +11001489
1490STATIC void
1491xfs_register_buftarg(
1492 xfs_buftarg_t *btp)
1493{
1494 spin_lock(&xfs_buftarg_lock);
1495 list_add(&btp->bt_list, &xfs_buftarg_list);
1496 spin_unlock(&xfs_buftarg_lock);
1497}
1498
1499STATIC void
1500xfs_unregister_buftarg(
1501 xfs_buftarg_t *btp)
1502{
1503 spin_lock(&xfs_buftarg_lock);
1504 list_del(&btp->bt_list);
1505 spin_unlock(&xfs_buftarg_lock);
1506}
1507
Linus Torvalds1da177e2005-04-16 15:20:36 -07001508void
1509xfs_free_buftarg(
Christoph Hellwigb7963132009-03-03 14:48:37 -05001510 struct xfs_mount *mp,
1511 struct xfs_buftarg *btp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001512{
1513 xfs_flush_buftarg(btp, 1);
Christoph Hellwigb7963132009-03-03 14:48:37 -05001514 if (mp->m_flags & XFS_MOUNT_BARRIER)
1515 xfs_blkdev_issue_flush(btp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001516 xfs_free_bufhash(btp);
Nathan Scottce8e9222006-01-11 15:39:08 +11001517 iput(btp->bt_mapping->host);
David Chinnera6867a62006-01-11 15:37:58 +11001518
Nathan Scottce8e9222006-01-11 15:39:08 +11001519 /* Unregister the buftarg first so that we don't get a
1520 * wakeup finding a non-existent task
1521 */
David Chinnera6867a62006-01-11 15:37:58 +11001522 xfs_unregister_buftarg(btp);
1523 kthread_stop(btp->bt_task);
1524
Denys Vlasenkof0e2d932008-05-19 16:31:57 +10001525 kmem_free(btp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001526}
1527
Linus Torvalds1da177e2005-04-16 15:20:36 -07001528STATIC int
1529xfs_setsize_buftarg_flags(
1530 xfs_buftarg_t *btp,
1531 unsigned int blocksize,
1532 unsigned int sectorsize,
1533 int verbose)
1534{
Nathan Scottce8e9222006-01-11 15:39:08 +11001535 btp->bt_bsize = blocksize;
1536 btp->bt_sshift = ffs(sectorsize) - 1;
1537 btp->bt_smask = sectorsize - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001538
Nathan Scottce8e9222006-01-11 15:39:08 +11001539 if (set_blocksize(btp->bt_bdev, sectorsize)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001540 printk(KERN_WARNING
1541 "XFS: Cannot set_blocksize to %u on device %s\n",
1542 sectorsize, XFS_BUFTARG_NAME(btp));
1543 return EINVAL;
1544 }
1545
1546 if (verbose &&
1547 (PAGE_CACHE_SIZE / BITS_PER_LONG) > sectorsize) {
1548 printk(KERN_WARNING
1549 "XFS: %u byte sectors in use on device %s. "
1550 "This is suboptimal; %u or greater is ideal.\n",
1551 sectorsize, XFS_BUFTARG_NAME(btp),
1552 (unsigned int)PAGE_CACHE_SIZE / BITS_PER_LONG);
1553 }
1554
1555 return 0;
1556}
1557
1558/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001559 * When allocating the initial buffer target we have not yet
1560 * read in the superblock, so don't know what sized sectors
1561 * are being used is at this early stage. Play safe.
1562 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001563STATIC int
1564xfs_setsize_buftarg_early(
1565 xfs_buftarg_t *btp,
1566 struct block_device *bdev)
1567{
1568 return xfs_setsize_buftarg_flags(btp,
Martin K. Petersene1defc42009-05-22 17:17:49 -04001569 PAGE_CACHE_SIZE, bdev_logical_block_size(bdev), 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001570}
1571
1572int
1573xfs_setsize_buftarg(
1574 xfs_buftarg_t *btp,
1575 unsigned int blocksize,
1576 unsigned int sectorsize)
1577{
1578 return xfs_setsize_buftarg_flags(btp, blocksize, sectorsize, 1);
1579}
1580
1581STATIC int
1582xfs_mapping_buftarg(
1583 xfs_buftarg_t *btp,
1584 struct block_device *bdev)
1585{
1586 struct backing_dev_info *bdi;
1587 struct inode *inode;
1588 struct address_space *mapping;
Christoph Hellwigf5e54d62006-06-28 04:26:44 -07001589 static const struct address_space_operations mapping_aops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001590 .sync_page = block_sync_page,
Christoph Lametere965f962006-02-01 03:05:41 -08001591 .migratepage = fail_migrate_page,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001592 };
1593
1594 inode = new_inode(bdev->bd_inode->i_sb);
1595 if (!inode) {
1596 printk(KERN_WARNING
1597 "XFS: Cannot allocate mapping inode for device %s\n",
1598 XFS_BUFTARG_NAME(btp));
1599 return ENOMEM;
1600 }
1601 inode->i_mode = S_IFBLK;
1602 inode->i_bdev = bdev;
1603 inode->i_rdev = bdev->bd_dev;
1604 bdi = blk_get_backing_dev_info(bdev);
1605 if (!bdi)
1606 bdi = &default_backing_dev_info;
1607 mapping = &inode->i_data;
1608 mapping->a_ops = &mapping_aops;
1609 mapping->backing_dev_info = bdi;
1610 mapping_set_gfp_mask(mapping, GFP_NOFS);
Nathan Scottce8e9222006-01-11 15:39:08 +11001611 btp->bt_mapping = mapping;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001612 return 0;
1613}
1614
David Chinnera6867a62006-01-11 15:37:58 +11001615STATIC int
1616xfs_alloc_delwrite_queue(
1617 xfs_buftarg_t *btp)
1618{
1619 int error = 0;
1620
1621 INIT_LIST_HEAD(&btp->bt_list);
1622 INIT_LIST_HEAD(&btp->bt_delwrite_queue);
Eric Sandeen007c61c2007-10-11 17:43:56 +10001623 spin_lock_init(&btp->bt_delwrite_lock);
David Chinnera6867a62006-01-11 15:37:58 +11001624 btp->bt_flags = 0;
1625 btp->bt_task = kthread_run(xfsbufd, btp, "xfsbufd");
1626 if (IS_ERR(btp->bt_task)) {
1627 error = PTR_ERR(btp->bt_task);
1628 goto out_error;
1629 }
1630 xfs_register_buftarg(btp);
1631out_error:
1632 return error;
1633}
1634
Linus Torvalds1da177e2005-04-16 15:20:36 -07001635xfs_buftarg_t *
1636xfs_alloc_buftarg(
1637 struct block_device *bdev,
1638 int external)
1639{
1640 xfs_buftarg_t *btp;
1641
1642 btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
1643
Nathan Scottce8e9222006-01-11 15:39:08 +11001644 btp->bt_dev = bdev->bd_dev;
1645 btp->bt_bdev = bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001646 if (xfs_setsize_buftarg_early(btp, bdev))
1647 goto error;
1648 if (xfs_mapping_buftarg(btp, bdev))
1649 goto error;
David Chinnera6867a62006-01-11 15:37:58 +11001650 if (xfs_alloc_delwrite_queue(btp))
1651 goto error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001652 xfs_alloc_bufhash(btp, external);
1653 return btp;
1654
1655error:
Denys Vlasenkof0e2d932008-05-19 16:31:57 +10001656 kmem_free(btp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001657 return NULL;
1658}
1659
1660
1661/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001662 * Delayed write buffer handling
Linus Torvalds1da177e2005-04-16 15:20:36 -07001663 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001664STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001665xfs_buf_delwri_queue(
1666 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001667 int unlock)
1668{
Nathan Scottce8e9222006-01-11 15:39:08 +11001669 struct list_head *dwq = &bp->b_target->bt_delwrite_queue;
1670 spinlock_t *dwlk = &bp->b_target->bt_delwrite_lock;
David Chinnera6867a62006-01-11 15:37:58 +11001671
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001672 trace_xfs_buf_delwri_queue(bp, _RET_IP_);
1673
Nathan Scottce8e9222006-01-11 15:39:08 +11001674 ASSERT((bp->b_flags&(XBF_DELWRI|XBF_ASYNC)) == (XBF_DELWRI|XBF_ASYNC));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001675
David Chinnera6867a62006-01-11 15:37:58 +11001676 spin_lock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001677 /* If already in the queue, dequeue and place at tail */
Nathan Scottce8e9222006-01-11 15:39:08 +11001678 if (!list_empty(&bp->b_list)) {
1679 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1680 if (unlock)
1681 atomic_dec(&bp->b_hold);
1682 list_del(&bp->b_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001683 }
1684
Dave Chinnerc9c12972010-01-11 11:49:59 +00001685 if (list_empty(dwq)) {
1686 /* start xfsbufd as it is about to have something to do */
1687 wake_up_process(bp->b_target->bt_task);
1688 }
1689
Nathan Scottce8e9222006-01-11 15:39:08 +11001690 bp->b_flags |= _XBF_DELWRI_Q;
1691 list_add_tail(&bp->b_list, dwq);
1692 bp->b_queuetime = jiffies;
David Chinnera6867a62006-01-11 15:37:58 +11001693 spin_unlock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001694
1695 if (unlock)
Nathan Scottce8e9222006-01-11 15:39:08 +11001696 xfs_buf_unlock(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001697}
1698
1699void
Nathan Scottce8e9222006-01-11 15:39:08 +11001700xfs_buf_delwri_dequeue(
1701 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001702{
Nathan Scottce8e9222006-01-11 15:39:08 +11001703 spinlock_t *dwlk = &bp->b_target->bt_delwrite_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001704 int dequeued = 0;
1705
David Chinnera6867a62006-01-11 15:37:58 +11001706 spin_lock(dwlk);
Nathan Scottce8e9222006-01-11 15:39:08 +11001707 if ((bp->b_flags & XBF_DELWRI) && !list_empty(&bp->b_list)) {
1708 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1709 list_del_init(&bp->b_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001710 dequeued = 1;
1711 }
Nathan Scottce8e9222006-01-11 15:39:08 +11001712 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
David Chinnera6867a62006-01-11 15:37:58 +11001713 spin_unlock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001714
1715 if (dequeued)
Nathan Scottce8e9222006-01-11 15:39:08 +11001716 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001717
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001718 trace_xfs_buf_delwri_dequeue(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001719}
1720
Dave Chinnerd808f612010-02-02 10:13:42 +11001721/*
1722 * If a delwri buffer needs to be pushed before it has aged out, then promote
1723 * it to the head of the delwri queue so that it will be flushed on the next
1724 * xfsbufd run. We do this by resetting the queuetime of the buffer to be older
1725 * than the age currently needed to flush the buffer. Hence the next time the
1726 * xfsbufd sees it is guaranteed to be considered old enough to flush.
1727 */
1728void
1729xfs_buf_delwri_promote(
1730 struct xfs_buf *bp)
1731{
1732 struct xfs_buftarg *btp = bp->b_target;
1733 long age = xfs_buf_age_centisecs * msecs_to_jiffies(10) + 1;
1734
1735 ASSERT(bp->b_flags & XBF_DELWRI);
1736 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1737
1738 /*
1739 * Check the buffer age before locking the delayed write queue as we
1740 * don't need to promote buffers that are already past the flush age.
1741 */
1742 if (bp->b_queuetime < jiffies - age)
1743 return;
1744 bp->b_queuetime = jiffies - age;
1745 spin_lock(&btp->bt_delwrite_lock);
1746 list_move(&bp->b_list, &btp->bt_delwrite_queue);
1747 spin_unlock(&btp->bt_delwrite_lock);
1748}
1749
Linus Torvalds1da177e2005-04-16 15:20:36 -07001750STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001751xfs_buf_runall_queues(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001752 struct workqueue_struct *queue)
1753{
1754 flush_workqueue(queue);
1755}
1756
Linus Torvalds1da177e2005-04-16 15:20:36 -07001757STATIC int
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001758xfsbufd_wakeup(
Nathan Scott15c84a42005-11-04 10:51:01 +11001759 int priority,
1760 gfp_t mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001761{
Christoph Hellwigda7f93e2006-01-11 20:49:57 +11001762 xfs_buftarg_t *btp;
David Chinnera6867a62006-01-11 15:37:58 +11001763
1764 spin_lock(&xfs_buftarg_lock);
Christoph Hellwigda7f93e2006-01-11 20:49:57 +11001765 list_for_each_entry(btp, &xfs_buftarg_list, bt_list) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001766 if (test_bit(XBT_FORCE_SLEEP, &btp->bt_flags))
David Chinnera6867a62006-01-11 15:37:58 +11001767 continue;
Dave Chinnerc9c12972010-01-11 11:49:59 +00001768 if (list_empty(&btp->bt_delwrite_queue))
1769 continue;
Nathan Scottce8e9222006-01-11 15:39:08 +11001770 set_bit(XBT_FORCE_FLUSH, &btp->bt_flags);
David Chinnera6867a62006-01-11 15:37:58 +11001771 wake_up_process(btp->bt_task);
1772 }
1773 spin_unlock(&xfs_buftarg_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001774 return 0;
1775}
1776
David Chinner585e6d82007-02-10 18:32:29 +11001777/*
1778 * Move as many buffers as specified to the supplied list
1779 * idicating if we skipped any buffers to prevent deadlocks.
1780 */
1781STATIC int
1782xfs_buf_delwri_split(
1783 xfs_buftarg_t *target,
1784 struct list_head *list,
David Chinner5e6a07d2007-02-10 18:34:49 +11001785 unsigned long age)
David Chinner585e6d82007-02-10 18:32:29 +11001786{
1787 xfs_buf_t *bp, *n;
1788 struct list_head *dwq = &target->bt_delwrite_queue;
1789 spinlock_t *dwlk = &target->bt_delwrite_lock;
1790 int skipped = 0;
David Chinner5e6a07d2007-02-10 18:34:49 +11001791 int force;
David Chinner585e6d82007-02-10 18:32:29 +11001792
David Chinner5e6a07d2007-02-10 18:34:49 +11001793 force = test_and_clear_bit(XBT_FORCE_FLUSH, &target->bt_flags);
David Chinner585e6d82007-02-10 18:32:29 +11001794 INIT_LIST_HEAD(list);
1795 spin_lock(dwlk);
1796 list_for_each_entry_safe(bp, n, dwq, b_list) {
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001797 trace_xfs_buf_delwri_split(bp, _RET_IP_);
David Chinner585e6d82007-02-10 18:32:29 +11001798 ASSERT(bp->b_flags & XBF_DELWRI);
1799
1800 if (!xfs_buf_ispin(bp) && !xfs_buf_cond_lock(bp)) {
David Chinner5e6a07d2007-02-10 18:34:49 +11001801 if (!force &&
David Chinner585e6d82007-02-10 18:32:29 +11001802 time_before(jiffies, bp->b_queuetime + age)) {
1803 xfs_buf_unlock(bp);
1804 break;
1805 }
1806
1807 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q|
1808 _XBF_RUN_QUEUES);
1809 bp->b_flags |= XBF_WRITE;
1810 list_move_tail(&bp->b_list, list);
1811 } else
1812 skipped++;
1813 }
1814 spin_unlock(dwlk);
1815
1816 return skipped;
1817
1818}
1819
Dave Chinner089716a2010-01-26 15:13:25 +11001820/*
1821 * Compare function is more complex than it needs to be because
1822 * the return value is only 32 bits and we are doing comparisons
1823 * on 64 bit values
1824 */
1825static int
1826xfs_buf_cmp(
1827 void *priv,
1828 struct list_head *a,
1829 struct list_head *b)
1830{
1831 struct xfs_buf *ap = container_of(a, struct xfs_buf, b_list);
1832 struct xfs_buf *bp = container_of(b, struct xfs_buf, b_list);
1833 xfs_daddr_t diff;
1834
1835 diff = ap->b_bn - bp->b_bn;
1836 if (diff < 0)
1837 return -1;
1838 if (diff > 0)
1839 return 1;
1840 return 0;
1841}
1842
1843void
1844xfs_buf_delwri_sort(
1845 xfs_buftarg_t *target,
1846 struct list_head *list)
1847{
1848 list_sort(NULL, list, xfs_buf_cmp);
1849}
1850
Linus Torvalds1da177e2005-04-16 15:20:36 -07001851STATIC int
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001852xfsbufd(
David Chinner585e6d82007-02-10 18:32:29 +11001853 void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001854{
Dave Chinner089716a2010-01-26 15:13:25 +11001855 xfs_buftarg_t *target = (xfs_buftarg_t *)data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001856
Linus Torvalds1da177e2005-04-16 15:20:36 -07001857 current->flags |= PF_MEMALLOC;
1858
Rafael J. Wysocki978c7b22007-12-07 14:09:02 +11001859 set_freezable();
1860
Linus Torvalds1da177e2005-04-16 15:20:36 -07001861 do {
Dave Chinnerc9c12972010-01-11 11:49:59 +00001862 long age = xfs_buf_age_centisecs * msecs_to_jiffies(10);
1863 long tout = xfs_buf_timer_centisecs * msecs_to_jiffies(10);
Dave Chinner089716a2010-01-26 15:13:25 +11001864 int count = 0;
1865 struct list_head tmp;
Dave Chinnerc9c12972010-01-11 11:49:59 +00001866
Christoph Lameter3e1d1d22005-06-24 23:13:50 -07001867 if (unlikely(freezing(current))) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001868 set_bit(XBT_FORCE_SLEEP, &target->bt_flags);
Christoph Lameter3e1d1d22005-06-24 23:13:50 -07001869 refrigerator();
Nathan Scottabd0cf72005-05-05 13:30:13 -07001870 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +11001871 clear_bit(XBT_FORCE_SLEEP, &target->bt_flags);
Nathan Scottabd0cf72005-05-05 13:30:13 -07001872 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001873
Dave Chinnerc9c12972010-01-11 11:49:59 +00001874 /* sleep for a long time if there is nothing to do. */
1875 if (list_empty(&target->bt_delwrite_queue))
1876 tout = MAX_SCHEDULE_TIMEOUT;
1877 schedule_timeout_interruptible(tout);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001878
Dave Chinnerc9c12972010-01-11 11:49:59 +00001879 xfs_buf_delwri_split(target, &tmp, age);
Dave Chinner089716a2010-01-26 15:13:25 +11001880 list_sort(NULL, &tmp, xfs_buf_cmp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001881 while (!list_empty(&tmp)) {
Dave Chinner089716a2010-01-26 15:13:25 +11001882 struct xfs_buf *bp;
1883 bp = list_first_entry(&tmp, struct xfs_buf, b_list);
Nathan Scottce8e9222006-01-11 15:39:08 +11001884 list_del_init(&bp->b_list);
1885 xfs_buf_iostrategy(bp);
David Chinner585e6d82007-02-10 18:32:29 +11001886 count++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001887 }
Nathan Scottf07c2252006-09-28 10:52:15 +10001888 if (count)
1889 blk_run_address_space(target->bt_mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001890
Christoph Hellwig4df08c52005-09-05 08:34:18 +10001891 } while (!kthread_should_stop());
Linus Torvalds1da177e2005-04-16 15:20:36 -07001892
Christoph Hellwig4df08c52005-09-05 08:34:18 +10001893 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001894}
1895
1896/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001897 * Go through all incore buffers, and release buffers if they belong to
1898 * the given device. This is used in filesystem error handling to
1899 * preserve the consistency of its metadata.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001900 */
1901int
1902xfs_flush_buftarg(
David Chinner585e6d82007-02-10 18:32:29 +11001903 xfs_buftarg_t *target,
1904 int wait)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001905{
Dave Chinner089716a2010-01-26 15:13:25 +11001906 xfs_buf_t *bp;
David Chinner585e6d82007-02-10 18:32:29 +11001907 int pincount = 0;
Dave Chinner089716a2010-01-26 15:13:25 +11001908 LIST_HEAD(tmp_list);
1909 LIST_HEAD(wait_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001910
Dave Chinnerc626d172009-04-06 18:42:11 +02001911 xfs_buf_runall_queues(xfsconvertd_workqueue);
Nathan Scottce8e9222006-01-11 15:39:08 +11001912 xfs_buf_runall_queues(xfsdatad_workqueue);
1913 xfs_buf_runall_queues(xfslogd_workqueue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001914
David Chinner5e6a07d2007-02-10 18:34:49 +11001915 set_bit(XBT_FORCE_FLUSH, &target->bt_flags);
Dave Chinner089716a2010-01-26 15:13:25 +11001916 pincount = xfs_buf_delwri_split(target, &tmp_list, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001917
1918 /*
Dave Chinner089716a2010-01-26 15:13:25 +11001919 * Dropped the delayed write list lock, now walk the temporary list.
1920 * All I/O is issued async and then if we need to wait for completion
1921 * we do that after issuing all the IO.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001922 */
Dave Chinner089716a2010-01-26 15:13:25 +11001923 list_sort(NULL, &tmp_list, xfs_buf_cmp);
1924 while (!list_empty(&tmp_list)) {
1925 bp = list_first_entry(&tmp_list, struct xfs_buf, b_list);
David Chinner585e6d82007-02-10 18:32:29 +11001926 ASSERT(target == bp->b_target);
Dave Chinner089716a2010-01-26 15:13:25 +11001927 list_del_init(&bp->b_list);
1928 if (wait) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001929 bp->b_flags &= ~XBF_ASYNC;
Dave Chinner089716a2010-01-26 15:13:25 +11001930 list_add(&bp->b_list, &wait_list);
1931 }
Nathan Scottce8e9222006-01-11 15:39:08 +11001932 xfs_buf_iostrategy(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001933 }
1934
Dave Chinner089716a2010-01-26 15:13:25 +11001935 if (wait) {
1936 /* Expedite and wait for IO to complete. */
Nathan Scottf07c2252006-09-28 10:52:15 +10001937 blk_run_address_space(target->bt_mapping);
Dave Chinner089716a2010-01-26 15:13:25 +11001938 while (!list_empty(&wait_list)) {
1939 bp = list_first_entry(&wait_list, struct xfs_buf, b_list);
Nathan Scottf07c2252006-09-28 10:52:15 +10001940
Dave Chinner089716a2010-01-26 15:13:25 +11001941 list_del_init(&bp->b_list);
1942 xfs_iowait(bp);
1943 xfs_buf_relse(bp);
1944 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001945 }
1946
Linus Torvalds1da177e2005-04-16 15:20:36 -07001947 return pincount;
1948}
1949
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001950int __init
Nathan Scottce8e9222006-01-11 15:39:08 +11001951xfs_buf_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001952{
Nathan Scott87582802006-03-14 13:18:19 +11001953 xfs_buf_zone = kmem_zone_init_flags(sizeof(xfs_buf_t), "xfs_buf",
1954 KM_ZONE_HWALIGN, NULL);
Nathan Scottce8e9222006-01-11 15:39:08 +11001955 if (!xfs_buf_zone)
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001956 goto out;
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001957
Rafael J. Wysockib4337692007-03-22 00:11:27 -08001958 xfslogd_workqueue = create_workqueue("xfslogd");
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001959 if (!xfslogd_workqueue)
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001960 goto out_free_buf_zone;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001961
Rafael J. Wysockib4337692007-03-22 00:11:27 -08001962 xfsdatad_workqueue = create_workqueue("xfsdatad");
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001963 if (!xfsdatad_workqueue)
1964 goto out_destroy_xfslogd_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001965
Dave Chinnerc626d172009-04-06 18:42:11 +02001966 xfsconvertd_workqueue = create_workqueue("xfsconvertd");
1967 if (!xfsconvertd_workqueue)
1968 goto out_destroy_xfsdatad_workqueue;
1969
Rusty Russell8e1f9362007-07-17 04:03:17 -07001970 register_shrinker(&xfs_buf_shake);
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001971 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001972
Dave Chinnerc626d172009-04-06 18:42:11 +02001973 out_destroy_xfsdatad_workqueue:
1974 destroy_workqueue(xfsdatad_workqueue);
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001975 out_destroy_xfslogd_workqueue:
1976 destroy_workqueue(xfslogd_workqueue);
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001977 out_free_buf_zone:
Nathan Scottce8e9222006-01-11 15:39:08 +11001978 kmem_zone_destroy(xfs_buf_zone);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001979 out:
Nathan Scott87582802006-03-14 13:18:19 +11001980 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001981}
1982
Linus Torvalds1da177e2005-04-16 15:20:36 -07001983void
Nathan Scottce8e9222006-01-11 15:39:08 +11001984xfs_buf_terminate(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001985{
Rusty Russell8e1f9362007-07-17 04:03:17 -07001986 unregister_shrinker(&xfs_buf_shake);
Dave Chinnerc626d172009-04-06 18:42:11 +02001987 destroy_workqueue(xfsconvertd_workqueue);
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001988 destroy_workqueue(xfsdatad_workqueue);
1989 destroy_workqueue(xfslogd_workqueue);
Nathan Scottce8e9222006-01-11 15:39:08 +11001990 kmem_zone_destroy(xfs_buf_zone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001991}
Tim Shimmine6a0e9c2007-05-08 13:49:59 +10001992
1993#ifdef CONFIG_KDB_MODULES
1994struct list_head *
1995xfs_get_buftarg_list(void)
1996{
1997 return &xfs_buftarg_list;
1998}
1999#endif