blob: 92f1f2acc6ab1db0ca894621ab86fcdd0e125d93 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Nathan Scottf07c2252006-09-28 10:52:15 +10002 * Copyright (c) 2000-2006 Silicon Graphics, Inc.
Nathan Scott7b718762005-11-02 14:58:39 +11003 * All Rights Reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
Nathan Scott7b718762005-11-02 14:58:39 +11005 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License as
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 * published by the Free Software Foundation.
8 *
Nathan Scott7b718762005-11-02 14:58:39 +11009 * This program is distributed in the hope that it would be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 *
Nathan Scott7b718762005-11-02 14:58:39 +110014 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
Linus Torvalds1da177e2005-04-16 15:20:36 -070017 */
Vlad Apostolov93c189c2006-11-11 18:03:49 +110018#include "xfs.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/stddef.h>
20#include <linux/errno.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090021#include <linux/gfp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022#include <linux/pagemap.h>
23#include <linux/init.h>
24#include <linux/vmalloc.h>
25#include <linux/bio.h>
26#include <linux/sysctl.h>
27#include <linux/proc_fs.h>
28#include <linux/workqueue.h>
29#include <linux/percpu.h>
30#include <linux/blkdev.h>
31#include <linux/hash.h>
Christoph Hellwig4df08c52005-09-05 08:34:18 +100032#include <linux/kthread.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080033#include <linux/migrate.h>
Andrew Morton3fcfab12006-10-19 23:28:16 -070034#include <linux/backing-dev.h>
Nigel Cunningham7dfb7102006-12-06 20:34:23 -080035#include <linux/freezer.h>
Dave Chinner089716a2010-01-26 15:13:25 +110036#include <linux/list_sort.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070037
Christoph Hellwigb7963132009-03-03 14:48:37 -050038#include "xfs_sb.h"
39#include "xfs_inum.h"
Dave Chinnered3b4d62010-05-21 12:07:08 +100040#include "xfs_log.h"
Christoph Hellwigb7963132009-03-03 14:48:37 -050041#include "xfs_ag.h"
Christoph Hellwigb7963132009-03-03 14:48:37 -050042#include "xfs_mount.h"
Christoph Hellwig0b1b2132009-12-14 23:14:59 +000043#include "xfs_trace.h"
Christoph Hellwigb7963132009-03-03 14:48:37 -050044
David Chinner7989cb82007-02-10 18:34:56 +110045static kmem_zone_t *xfs_buf_zone;
David Chinnera6867a62006-01-11 15:37:58 +110046STATIC int xfsbufd(void *);
Nathan Scottce8e9222006-01-11 15:39:08 +110047STATIC void xfs_buf_delwri_queue(xfs_buf_t *, int);
Christoph Hellwig23ea4032005-06-21 15:14:01 +100048
David Chinner7989cb82007-02-10 18:34:56 +110049static struct workqueue_struct *xfslogd_workqueue;
Christoph Hellwig0829c362005-09-02 16:58:49 +100050struct workqueue_struct *xfsdatad_workqueue;
Dave Chinnerc626d172009-04-06 18:42:11 +020051struct workqueue_struct *xfsconvertd_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070052
Nathan Scottce8e9222006-01-11 15:39:08 +110053#ifdef XFS_BUF_LOCK_TRACKING
54# define XB_SET_OWNER(bp) ((bp)->b_last_holder = current->pid)
55# define XB_CLEAR_OWNER(bp) ((bp)->b_last_holder = -1)
56# define XB_GET_OWNER(bp) ((bp)->b_last_holder)
Linus Torvalds1da177e2005-04-16 15:20:36 -070057#else
Nathan Scottce8e9222006-01-11 15:39:08 +110058# define XB_SET_OWNER(bp) do { } while (0)
59# define XB_CLEAR_OWNER(bp) do { } while (0)
60# define XB_GET_OWNER(bp) do { } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -070061#endif
62
Nathan Scottce8e9222006-01-11 15:39:08 +110063#define xb_to_gfp(flags) \
64 ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : \
65 ((flags) & XBF_DONT_BLOCK) ? GFP_NOFS : GFP_KERNEL) | __GFP_NOWARN)
Linus Torvalds1da177e2005-04-16 15:20:36 -070066
Nathan Scottce8e9222006-01-11 15:39:08 +110067#define xb_to_km(flags) \
68 (((flags) & XBF_DONT_BLOCK) ? KM_NOFS : KM_SLEEP)
Linus Torvalds1da177e2005-04-16 15:20:36 -070069
Nathan Scottce8e9222006-01-11 15:39:08 +110070#define xfs_buf_allocate(flags) \
71 kmem_zone_alloc(xfs_buf_zone, xb_to_km(flags))
72#define xfs_buf_deallocate(bp) \
73 kmem_zone_free(xfs_buf_zone, (bp));
Linus Torvalds1da177e2005-04-16 15:20:36 -070074
James Bottomley73c77e22010-01-25 11:42:24 -060075static inline int
76xfs_buf_is_vmapped(
77 struct xfs_buf *bp)
78{
79 /*
80 * Return true if the buffer is vmapped.
81 *
82 * The XBF_MAPPED flag is set if the buffer should be mapped, but the
83 * code is clever enough to know it doesn't have to map a single page,
84 * so the check has to be both for XBF_MAPPED and bp->b_page_count > 1.
85 */
86 return (bp->b_flags & XBF_MAPPED) && bp->b_page_count > 1;
87}
88
89static inline int
90xfs_buf_vmap_len(
91 struct xfs_buf *bp)
92{
93 return (bp->b_page_count * PAGE_SIZE) - bp->b_offset;
94}
95
Linus Torvalds1da177e2005-04-16 15:20:36 -070096/*
Nathan Scottce8e9222006-01-11 15:39:08 +110097 * Page Region interfaces.
Linus Torvalds1da177e2005-04-16 15:20:36 -070098 *
Nathan Scottce8e9222006-01-11 15:39:08 +110099 * For pages in filesystems where the blocksize is smaller than the
100 * pagesize, we use the page->private field (long) to hold a bitmap
101 * of uptodate regions within the page.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102 *
Nathan Scottce8e9222006-01-11 15:39:08 +1100103 * Each such region is "bytes per page / bits per long" bytes long.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104 *
Nathan Scottce8e9222006-01-11 15:39:08 +1100105 * NBPPR == number-of-bytes-per-page-region
106 * BTOPR == bytes-to-page-region (rounded up)
107 * BTOPRT == bytes-to-page-region-truncated (rounded down)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108 */
109#if (BITS_PER_LONG == 32)
110#define PRSHIFT (PAGE_CACHE_SHIFT - 5) /* (32 == 1<<5) */
111#elif (BITS_PER_LONG == 64)
112#define PRSHIFT (PAGE_CACHE_SHIFT - 6) /* (64 == 1<<6) */
113#else
114#error BITS_PER_LONG must be 32 or 64
115#endif
116#define NBPPR (PAGE_CACHE_SIZE/BITS_PER_LONG)
117#define BTOPR(b) (((unsigned int)(b) + (NBPPR - 1)) >> PRSHIFT)
118#define BTOPRT(b) (((unsigned int)(b) >> PRSHIFT))
119
120STATIC unsigned long
121page_region_mask(
122 size_t offset,
123 size_t length)
124{
125 unsigned long mask;
126 int first, final;
127
128 first = BTOPR(offset);
129 final = BTOPRT(offset + length - 1);
130 first = min(first, final);
131
132 mask = ~0UL;
133 mask <<= BITS_PER_LONG - (final - first);
134 mask >>= BITS_PER_LONG - (final);
135
136 ASSERT(offset + length <= PAGE_CACHE_SIZE);
137 ASSERT((final - first) < BITS_PER_LONG && (final - first) >= 0);
138
139 return mask;
140}
141
Christoph Hellwigb8f82a42009-11-14 16:17:22 +0000142STATIC void
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143set_page_region(
144 struct page *page,
145 size_t offset,
146 size_t length)
147{
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700148 set_page_private(page,
149 page_private(page) | page_region_mask(offset, length));
150 if (page_private(page) == ~0UL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151 SetPageUptodate(page);
152}
153
Christoph Hellwigb8f82a42009-11-14 16:17:22 +0000154STATIC int
Linus Torvalds1da177e2005-04-16 15:20:36 -0700155test_page_region(
156 struct page *page,
157 size_t offset,
158 size_t length)
159{
160 unsigned long mask = page_region_mask(offset, length);
161
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700162 return (mask && (page_private(page) & mask) == mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163}
164
165/*
Dave Chinner430cbeb2010-12-02 16:30:55 +1100166 * xfs_buf_lru_add - add a buffer to the LRU.
167 *
168 * The LRU takes a new reference to the buffer so that it will only be freed
169 * once the shrinker takes the buffer off the LRU.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170 */
Dave Chinner430cbeb2010-12-02 16:30:55 +1100171STATIC void
172xfs_buf_lru_add(
173 struct xfs_buf *bp)
174{
175 struct xfs_buftarg *btp = bp->b_target;
176
177 spin_lock(&btp->bt_lru_lock);
178 if (list_empty(&bp->b_lru)) {
179 atomic_inc(&bp->b_hold);
180 list_add_tail(&bp->b_lru, &btp->bt_lru);
181 btp->bt_lru_nr++;
182 }
183 spin_unlock(&btp->bt_lru_lock);
184}
185
186/*
187 * xfs_buf_lru_del - remove a buffer from the LRU
188 *
189 * The unlocked check is safe here because it only occurs when there are not
190 * b_lru_ref counts left on the inode under the pag->pag_buf_lock. it is there
191 * to optimise the shrinker removing the buffer from the LRU and calling
192 * xfs_buf_free(). i.e. it removes an unneccessary round trip on the
193 * bt_lru_lock.
194 */
195STATIC void
196xfs_buf_lru_del(
197 struct xfs_buf *bp)
198{
199 struct xfs_buftarg *btp = bp->b_target;
200
201 if (list_empty(&bp->b_lru))
202 return;
203
204 spin_lock(&btp->bt_lru_lock);
205 if (!list_empty(&bp->b_lru)) {
206 list_del_init(&bp->b_lru);
207 btp->bt_lru_nr--;
208 }
209 spin_unlock(&btp->bt_lru_lock);
210}
211
212/*
213 * When we mark a buffer stale, we remove the buffer from the LRU and clear the
214 * b_lru_ref count so that the buffer is freed immediately when the buffer
215 * reference count falls to zero. If the buffer is already on the LRU, we need
216 * to remove the reference that LRU holds on the buffer.
217 *
218 * This prevents build-up of stale buffers on the LRU.
219 */
220void
221xfs_buf_stale(
222 struct xfs_buf *bp)
223{
224 bp->b_flags |= XBF_STALE;
225 atomic_set(&(bp)->b_lru_ref, 0);
226 if (!list_empty(&bp->b_lru)) {
227 struct xfs_buftarg *btp = bp->b_target;
228
229 spin_lock(&btp->bt_lru_lock);
230 if (!list_empty(&bp->b_lru)) {
231 list_del_init(&bp->b_lru);
232 btp->bt_lru_nr--;
233 atomic_dec(&bp->b_hold);
234 }
235 spin_unlock(&btp->bt_lru_lock);
236 }
237 ASSERT(atomic_read(&bp->b_hold) >= 1);
238}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239
240STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +1100241_xfs_buf_initialize(
242 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243 xfs_buftarg_t *target,
Nathan Scott204ab252006-01-11 20:50:22 +1100244 xfs_off_t range_base,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245 size_t range_length,
Nathan Scottce8e9222006-01-11 15:39:08 +1100246 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247{
248 /*
Nathan Scottce8e9222006-01-11 15:39:08 +1100249 * We don't want certain flags to appear in b_flags.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100251 flags &= ~(XBF_LOCK|XBF_MAPPED|XBF_DONT_BLOCK|XBF_READ_AHEAD);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700252
Nathan Scottce8e9222006-01-11 15:39:08 +1100253 memset(bp, 0, sizeof(xfs_buf_t));
254 atomic_set(&bp->b_hold, 1);
Dave Chinner430cbeb2010-12-02 16:30:55 +1100255 atomic_set(&bp->b_lru_ref, 1);
David Chinnerb4dd3302008-08-13 16:36:11 +1000256 init_completion(&bp->b_iowait);
Dave Chinner430cbeb2010-12-02 16:30:55 +1100257 INIT_LIST_HEAD(&bp->b_lru);
Nathan Scottce8e9222006-01-11 15:39:08 +1100258 INIT_LIST_HEAD(&bp->b_list);
Dave Chinner74f75a02010-09-24 19:59:04 +1000259 RB_CLEAR_NODE(&bp->b_rbnode);
Thomas Gleixnera731cd12010-09-07 14:33:15 +0000260 sema_init(&bp->b_sema, 0); /* held, no waiters */
Nathan Scottce8e9222006-01-11 15:39:08 +1100261 XB_SET_OWNER(bp);
262 bp->b_target = target;
263 bp->b_file_offset = range_base;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700264 /*
265 * Set buffer_length and count_desired to the same value initially.
266 * I/O routines should use count_desired, which will be the same in
267 * most cases but may be reset (e.g. XFS recovery).
268 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100269 bp->b_buffer_length = bp->b_count_desired = range_length;
270 bp->b_flags = flags;
271 bp->b_bn = XFS_BUF_DADDR_NULL;
272 atomic_set(&bp->b_pin_count, 0);
273 init_waitqueue_head(&bp->b_waiters);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274
Nathan Scottce8e9222006-01-11 15:39:08 +1100275 XFS_STATS_INC(xb_create);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000276
277 trace_xfs_buf_init(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278}
279
280/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100281 * Allocate a page array capable of holding a specified number
282 * of pages, and point the page buf at it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700283 */
284STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100285_xfs_buf_get_pages(
286 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287 int page_count,
Nathan Scottce8e9222006-01-11 15:39:08 +1100288 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700289{
290 /* Make sure that we have a page list */
Nathan Scottce8e9222006-01-11 15:39:08 +1100291 if (bp->b_pages == NULL) {
292 bp->b_offset = xfs_buf_poff(bp->b_file_offset);
293 bp->b_page_count = page_count;
294 if (page_count <= XB_PAGES) {
295 bp->b_pages = bp->b_page_array;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700296 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100297 bp->b_pages = kmem_alloc(sizeof(struct page *) *
298 page_count, xb_to_km(flags));
299 if (bp->b_pages == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300 return -ENOMEM;
301 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100302 memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303 }
304 return 0;
305}
306
307/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100308 * Frees b_pages if it was allocated.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309 */
310STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +1100311_xfs_buf_free_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312 xfs_buf_t *bp)
313{
Nathan Scottce8e9222006-01-11 15:39:08 +1100314 if (bp->b_pages != bp->b_page_array) {
Denys Vlasenkof0e2d932008-05-19 16:31:57 +1000315 kmem_free(bp->b_pages);
Dave Chinner3fc98b12009-12-14 23:11:57 +0000316 bp->b_pages = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317 }
318}
319
320/*
321 * Releases the specified buffer.
322 *
323 * The modification state of any associated pages is left unchanged.
Nathan Scottce8e9222006-01-11 15:39:08 +1100324 * The buffer most not be on any hash - use xfs_buf_rele instead for
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325 * hashed and refcounted buffers
326 */
327void
Nathan Scottce8e9222006-01-11 15:39:08 +1100328xfs_buf_free(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700329 xfs_buf_t *bp)
330{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000331 trace_xfs_buf_free(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332
Dave Chinner430cbeb2010-12-02 16:30:55 +1100333 ASSERT(list_empty(&bp->b_lru));
334
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000335 if (bp->b_flags & (_XBF_PAGE_CACHE|_XBF_PAGES)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336 uint i;
337
James Bottomley73c77e22010-01-25 11:42:24 -0600338 if (xfs_buf_is_vmapped(bp))
Alex Elder8a262e52010-03-16 18:55:56 +0000339 vm_unmap_ram(bp->b_addr - bp->b_offset,
340 bp->b_page_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700341
Nathan Scott948ecdb2006-09-28 11:03:13 +1000342 for (i = 0; i < bp->b_page_count; i++) {
343 struct page *page = bp->b_pages[i];
344
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000345 if (bp->b_flags & _XBF_PAGE_CACHE)
346 ASSERT(!PagePrivate(page));
Nathan Scott948ecdb2006-09-28 11:03:13 +1000347 page_cache_release(page);
348 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700349 }
Dave Chinner3fc98b12009-12-14 23:11:57 +0000350 _xfs_buf_free_pages(bp);
Nathan Scottce8e9222006-01-11 15:39:08 +1100351 xfs_buf_deallocate(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700352}
353
354/*
355 * Finds all pages for buffer in question and builds it's page list.
356 */
357STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100358_xfs_buf_lookup_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700359 xfs_buf_t *bp,
360 uint flags)
361{
Nathan Scottce8e9222006-01-11 15:39:08 +1100362 struct address_space *mapping = bp->b_target->bt_mapping;
363 size_t blocksize = bp->b_target->bt_bsize;
364 size_t size = bp->b_count_desired;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700365 size_t nbytes, offset;
Nathan Scottce8e9222006-01-11 15:39:08 +1100366 gfp_t gfp_mask = xb_to_gfp(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700367 unsigned short page_count, i;
368 pgoff_t first;
Nathan Scott204ab252006-01-11 20:50:22 +1100369 xfs_off_t end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370 int error;
371
Nathan Scottce8e9222006-01-11 15:39:08 +1100372 end = bp->b_file_offset + bp->b_buffer_length;
373 page_count = xfs_buf_btoc(end) - xfs_buf_btoct(bp->b_file_offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374
Nathan Scottce8e9222006-01-11 15:39:08 +1100375 error = _xfs_buf_get_pages(bp, page_count, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376 if (unlikely(error))
377 return error;
Nathan Scottce8e9222006-01-11 15:39:08 +1100378 bp->b_flags |= _XBF_PAGE_CACHE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379
Nathan Scottce8e9222006-01-11 15:39:08 +1100380 offset = bp->b_offset;
381 first = bp->b_file_offset >> PAGE_CACHE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382
Nathan Scottce8e9222006-01-11 15:39:08 +1100383 for (i = 0; i < bp->b_page_count; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384 struct page *page;
385 uint retries = 0;
386
387 retry:
388 page = find_or_create_page(mapping, first + i, gfp_mask);
389 if (unlikely(page == NULL)) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100390 if (flags & XBF_READ_AHEAD) {
391 bp->b_page_count = i;
Christoph Hellwig6ab455e2008-05-19 16:34:42 +1000392 for (i = 0; i < bp->b_page_count; i++)
393 unlock_page(bp->b_pages[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394 return -ENOMEM;
395 }
396
397 /*
398 * This could deadlock.
399 *
400 * But until all the XFS lowlevel code is revamped to
401 * handle buffer allocation failures we can't do much.
402 */
403 if (!(++retries % 100))
404 printk(KERN_ERR
405 "XFS: possible memory allocation "
406 "deadlock in %s (mode:0x%x)\n",
Harvey Harrison34a622b2008-04-10 12:19:21 +1000407 __func__, gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408
Nathan Scottce8e9222006-01-11 15:39:08 +1100409 XFS_STATS_INC(xb_page_retries);
Jens Axboe8aa7e842009-07-09 14:52:32 +0200410 congestion_wait(BLK_RW_ASYNC, HZ/50);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411 goto retry;
412 }
413
Nathan Scottce8e9222006-01-11 15:39:08 +1100414 XFS_STATS_INC(xb_page_found);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415
416 nbytes = min_t(size_t, size, PAGE_CACHE_SIZE - offset);
417 size -= nbytes;
418
Nathan Scott948ecdb2006-09-28 11:03:13 +1000419 ASSERT(!PagePrivate(page));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420 if (!PageUptodate(page)) {
421 page_count--;
Christoph Hellwig6ab455e2008-05-19 16:34:42 +1000422 if (blocksize >= PAGE_CACHE_SIZE) {
423 if (flags & XBF_READ)
424 bp->b_flags |= _XBF_PAGE_LOCKED;
425 } else if (!PagePrivate(page)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426 if (test_page_region(page, offset, nbytes))
427 page_count++;
428 }
429 }
430
Nathan Scottce8e9222006-01-11 15:39:08 +1100431 bp->b_pages[i] = page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432 offset = 0;
433 }
434
Christoph Hellwig6ab455e2008-05-19 16:34:42 +1000435 if (!(bp->b_flags & _XBF_PAGE_LOCKED)) {
436 for (i = 0; i < bp->b_page_count; i++)
437 unlock_page(bp->b_pages[i]);
438 }
439
Nathan Scottce8e9222006-01-11 15:39:08 +1100440 if (page_count == bp->b_page_count)
441 bp->b_flags |= XBF_DONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700442
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443 return error;
444}
445
446/*
447 * Map buffer into kernel address-space if nessecary.
448 */
449STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100450_xfs_buf_map_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700451 xfs_buf_t *bp,
452 uint flags)
453{
454 /* A single page buffer is always mappable */
Nathan Scottce8e9222006-01-11 15:39:08 +1100455 if (bp->b_page_count == 1) {
456 bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
457 bp->b_flags |= XBF_MAPPED;
458 } else if (flags & XBF_MAPPED) {
Alex Elder8a262e52010-03-16 18:55:56 +0000459 bp->b_addr = vm_map_ram(bp->b_pages, bp->b_page_count,
460 -1, PAGE_KERNEL);
Nathan Scottce8e9222006-01-11 15:39:08 +1100461 if (unlikely(bp->b_addr == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700462 return -ENOMEM;
Nathan Scottce8e9222006-01-11 15:39:08 +1100463 bp->b_addr += bp->b_offset;
464 bp->b_flags |= XBF_MAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465 }
466
467 return 0;
468}
469
470/*
471 * Finding and Reading Buffers
472 */
473
474/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100475 * Look up, and creates if absent, a lockable buffer for
Linus Torvalds1da177e2005-04-16 15:20:36 -0700476 * a given range of an inode. The buffer is returned
477 * locked. If other overlapping buffers exist, they are
478 * released before the new buffer is created and locked,
479 * which may imply that this call will block until those buffers
480 * are unlocked. No I/O is implied by this call.
481 */
482xfs_buf_t *
Nathan Scottce8e9222006-01-11 15:39:08 +1100483_xfs_buf_find(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700484 xfs_buftarg_t *btp, /* block device target */
Nathan Scott204ab252006-01-11 20:50:22 +1100485 xfs_off_t ioff, /* starting offset of range */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486 size_t isize, /* length of range */
Nathan Scottce8e9222006-01-11 15:39:08 +1100487 xfs_buf_flags_t flags,
488 xfs_buf_t *new_bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700489{
Nathan Scott204ab252006-01-11 20:50:22 +1100490 xfs_off_t range_base;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700491 size_t range_length;
Dave Chinner74f75a02010-09-24 19:59:04 +1000492 struct xfs_perag *pag;
493 struct rb_node **rbp;
494 struct rb_node *parent;
495 xfs_buf_t *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700496
497 range_base = (ioff << BBSHIFT);
498 range_length = (isize << BBSHIFT);
499
500 /* Check for IOs smaller than the sector size / not sector aligned */
Nathan Scottce8e9222006-01-11 15:39:08 +1100501 ASSERT(!(range_length < (1 << btp->bt_sshift)));
Nathan Scott204ab252006-01-11 20:50:22 +1100502 ASSERT(!(range_base & (xfs_off_t)btp->bt_smask));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503
Dave Chinner74f75a02010-09-24 19:59:04 +1000504 /* get tree root */
505 pag = xfs_perag_get(btp->bt_mount,
506 xfs_daddr_to_agno(btp->bt_mount, ioff));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507
Dave Chinner74f75a02010-09-24 19:59:04 +1000508 /* walk tree */
509 spin_lock(&pag->pag_buf_lock);
510 rbp = &pag->pag_buf_tree.rb_node;
511 parent = NULL;
512 bp = NULL;
513 while (*rbp) {
514 parent = *rbp;
515 bp = rb_entry(parent, struct xfs_buf, b_rbnode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516
Dave Chinner74f75a02010-09-24 19:59:04 +1000517 if (range_base < bp->b_file_offset)
518 rbp = &(*rbp)->rb_left;
519 else if (range_base > bp->b_file_offset)
520 rbp = &(*rbp)->rb_right;
521 else {
522 /*
523 * found a block offset match. If the range doesn't
524 * match, the only way this is allowed is if the buffer
525 * in the cache is stale and the transaction that made
526 * it stale has not yet committed. i.e. we are
527 * reallocating a busy extent. Skip this buffer and
528 * continue searching to the right for an exact match.
529 */
530 if (bp->b_buffer_length != range_length) {
531 ASSERT(bp->b_flags & XBF_STALE);
532 rbp = &(*rbp)->rb_right;
533 continue;
534 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100535 atomic_inc(&bp->b_hold);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700536 goto found;
537 }
538 }
539
540 /* No match found */
Nathan Scottce8e9222006-01-11 15:39:08 +1100541 if (new_bp) {
542 _xfs_buf_initialize(new_bp, btp, range_base,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543 range_length, flags);
Dave Chinner74f75a02010-09-24 19:59:04 +1000544 rb_link_node(&new_bp->b_rbnode, parent, rbp);
545 rb_insert_color(&new_bp->b_rbnode, &pag->pag_buf_tree);
546 /* the buffer keeps the perag reference until it is freed */
547 new_bp->b_pag = pag;
548 spin_unlock(&pag->pag_buf_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700549 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100550 XFS_STATS_INC(xb_miss_locked);
Dave Chinner74f75a02010-09-24 19:59:04 +1000551 spin_unlock(&pag->pag_buf_lock);
552 xfs_perag_put(pag);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700553 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100554 return new_bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700555
556found:
Dave Chinner74f75a02010-09-24 19:59:04 +1000557 spin_unlock(&pag->pag_buf_lock);
558 xfs_perag_put(pag);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700559
Dave Chinner90810b92010-11-30 15:16:16 +1100560 if (xfs_buf_cond_lock(bp)) {
561 /* failed, so wait for the lock if requested. */
Nathan Scottce8e9222006-01-11 15:39:08 +1100562 if (!(flags & XBF_TRYLOCK)) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100563 xfs_buf_lock(bp);
564 XFS_STATS_INC(xb_get_locked_waited);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100566 xfs_buf_rele(bp);
567 XFS_STATS_INC(xb_busy_locked);
568 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570 }
571
Nathan Scottce8e9222006-01-11 15:39:08 +1100572 if (bp->b_flags & XBF_STALE) {
573 ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
574 bp->b_flags &= XBF_MAPPED;
David Chinner2f926582005-09-05 08:33:35 +1000575 }
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000576
577 trace_xfs_buf_find(bp, flags, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +1100578 XFS_STATS_INC(xb_get_locked);
579 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580}
581
582/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100583 * Assembles a buffer covering the specified range.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584 * Storage in memory for all portions of the buffer will be allocated,
585 * although backing storage may not be.
586 */
587xfs_buf_t *
Christoph Hellwig6ad112b2009-11-24 18:02:23 +0000588xfs_buf_get(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589 xfs_buftarg_t *target,/* target for buffer */
Nathan Scott204ab252006-01-11 20:50:22 +1100590 xfs_off_t ioff, /* starting offset of range */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700591 size_t isize, /* length of range */
Nathan Scottce8e9222006-01-11 15:39:08 +1100592 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700593{
Nathan Scottce8e9222006-01-11 15:39:08 +1100594 xfs_buf_t *bp, *new_bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595 int error = 0, i;
596
Nathan Scottce8e9222006-01-11 15:39:08 +1100597 new_bp = xfs_buf_allocate(flags);
598 if (unlikely(!new_bp))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700599 return NULL;
600
Nathan Scottce8e9222006-01-11 15:39:08 +1100601 bp = _xfs_buf_find(target, ioff, isize, flags, new_bp);
602 if (bp == new_bp) {
603 error = _xfs_buf_lookup_pages(bp, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700604 if (error)
605 goto no_buffer;
606 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100607 xfs_buf_deallocate(new_bp);
608 if (unlikely(bp == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700609 return NULL;
610 }
611
Nathan Scottce8e9222006-01-11 15:39:08 +1100612 for (i = 0; i < bp->b_page_count; i++)
613 mark_page_accessed(bp->b_pages[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700614
Nathan Scottce8e9222006-01-11 15:39:08 +1100615 if (!(bp->b_flags & XBF_MAPPED)) {
616 error = _xfs_buf_map_pages(bp, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617 if (unlikely(error)) {
618 printk(KERN_WARNING "%s: failed to map pages\n",
Harvey Harrison34a622b2008-04-10 12:19:21 +1000619 __func__);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700620 goto no_buffer;
621 }
622 }
623
Nathan Scottce8e9222006-01-11 15:39:08 +1100624 XFS_STATS_INC(xb_get);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700625
626 /*
627 * Always fill in the block number now, the mapped cases can do
628 * their own overlay of this later.
629 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100630 bp->b_bn = ioff;
631 bp->b_count_desired = bp->b_buffer_length;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700632
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000633 trace_xfs_buf_get(bp, flags, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +1100634 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700635
636 no_buffer:
Nathan Scottce8e9222006-01-11 15:39:08 +1100637 if (flags & (XBF_LOCK | XBF_TRYLOCK))
638 xfs_buf_unlock(bp);
639 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700640 return NULL;
641}
642
Christoph Hellwig5d765b92008-12-03 12:20:26 +0100643STATIC int
644_xfs_buf_read(
645 xfs_buf_t *bp,
646 xfs_buf_flags_t flags)
647{
648 int status;
649
Christoph Hellwig5d765b92008-12-03 12:20:26 +0100650 ASSERT(!(flags & (XBF_DELWRI|XBF_WRITE)));
651 ASSERT(bp->b_bn != XFS_BUF_DADDR_NULL);
652
653 bp->b_flags &= ~(XBF_WRITE | XBF_ASYNC | XBF_DELWRI | \
654 XBF_READ_AHEAD | _XBF_RUN_QUEUES);
655 bp->b_flags |= flags & (XBF_READ | XBF_ASYNC | \
656 XBF_READ_AHEAD | _XBF_RUN_QUEUES);
657
658 status = xfs_buf_iorequest(bp);
Dave Chinnerec53d1d2010-07-20 17:52:59 +1000659 if (status || XFS_BUF_ISERROR(bp) || (flags & XBF_ASYNC))
660 return status;
661 return xfs_buf_iowait(bp);
Christoph Hellwig5d765b92008-12-03 12:20:26 +0100662}
663
Linus Torvalds1da177e2005-04-16 15:20:36 -0700664xfs_buf_t *
Christoph Hellwig6ad112b2009-11-24 18:02:23 +0000665xfs_buf_read(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666 xfs_buftarg_t *target,
Nathan Scott204ab252006-01-11 20:50:22 +1100667 xfs_off_t ioff,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668 size_t isize,
Nathan Scottce8e9222006-01-11 15:39:08 +1100669 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700670{
Nathan Scottce8e9222006-01-11 15:39:08 +1100671 xfs_buf_t *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700672
Nathan Scottce8e9222006-01-11 15:39:08 +1100673 flags |= XBF_READ;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700674
Christoph Hellwig6ad112b2009-11-24 18:02:23 +0000675 bp = xfs_buf_get(target, ioff, isize, flags);
Nathan Scottce8e9222006-01-11 15:39:08 +1100676 if (bp) {
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000677 trace_xfs_buf_read(bp, flags, _RET_IP_);
678
Nathan Scottce8e9222006-01-11 15:39:08 +1100679 if (!XFS_BUF_ISDONE(bp)) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100680 XFS_STATS_INC(xb_get_read);
Christoph Hellwig5d765b92008-12-03 12:20:26 +0100681 _xfs_buf_read(bp, flags);
Nathan Scottce8e9222006-01-11 15:39:08 +1100682 } else if (flags & XBF_ASYNC) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683 /*
684 * Read ahead call which is already satisfied,
685 * drop the buffer
686 */
687 goto no_buffer;
688 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700689 /* We do not want read in the flags */
Nathan Scottce8e9222006-01-11 15:39:08 +1100690 bp->b_flags &= ~XBF_READ;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700691 }
692 }
693
Nathan Scottce8e9222006-01-11 15:39:08 +1100694 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700695
696 no_buffer:
Nathan Scottce8e9222006-01-11 15:39:08 +1100697 if (flags & (XBF_LOCK | XBF_TRYLOCK))
698 xfs_buf_unlock(bp);
699 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700700 return NULL;
701}
702
703/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100704 * If we are not low on memory then do the readahead in a deadlock
705 * safe manner.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706 */
707void
Nathan Scottce8e9222006-01-11 15:39:08 +1100708xfs_buf_readahead(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700709 xfs_buftarg_t *target,
Nathan Scott204ab252006-01-11 20:50:22 +1100710 xfs_off_t ioff,
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +0000711 size_t isize)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700712{
713 struct backing_dev_info *bdi;
714
Nathan Scottce8e9222006-01-11 15:39:08 +1100715 bdi = target->bt_mapping->backing_dev_info;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700716 if (bdi_read_congested(bdi))
717 return;
718
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +0000719 xfs_buf_read(target, ioff, isize,
720 XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD|XBF_DONT_BLOCK);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700721}
722
Dave Chinner5adc94c2010-09-24 21:58:31 +1000723/*
724 * Read an uncached buffer from disk. Allocates and returns a locked
725 * buffer containing the disk contents or nothing.
726 */
727struct xfs_buf *
728xfs_buf_read_uncached(
729 struct xfs_mount *mp,
730 struct xfs_buftarg *target,
731 xfs_daddr_t daddr,
732 size_t length,
733 int flags)
734{
735 xfs_buf_t *bp;
736 int error;
737
738 bp = xfs_buf_get_uncached(target, length, flags);
739 if (!bp)
740 return NULL;
741
742 /* set up the buffer for a read IO */
743 xfs_buf_lock(bp);
744 XFS_BUF_SET_ADDR(bp, daddr);
745 XFS_BUF_READ(bp);
746 XFS_BUF_BUSY(bp);
747
748 xfsbdstrat(mp, bp);
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +0000749 error = xfs_buf_iowait(bp);
Dave Chinner5adc94c2010-09-24 21:58:31 +1000750 if (error || bp->b_error) {
751 xfs_buf_relse(bp);
752 return NULL;
753 }
754 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700755}
756
757xfs_buf_t *
Nathan Scottce8e9222006-01-11 15:39:08 +1100758xfs_buf_get_empty(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700759 size_t len,
760 xfs_buftarg_t *target)
761{
Nathan Scottce8e9222006-01-11 15:39:08 +1100762 xfs_buf_t *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700763
Nathan Scottce8e9222006-01-11 15:39:08 +1100764 bp = xfs_buf_allocate(0);
765 if (bp)
766 _xfs_buf_initialize(bp, target, 0, len, 0);
767 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700768}
769
770static inline struct page *
771mem_to_page(
772 void *addr)
773{
Christoph Lameter9e2779f2008-02-04 22:28:34 -0800774 if ((!is_vmalloc_addr(addr))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700775 return virt_to_page(addr);
776 } else {
777 return vmalloc_to_page(addr);
778 }
779}
780
781int
Nathan Scottce8e9222006-01-11 15:39:08 +1100782xfs_buf_associate_memory(
783 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700784 void *mem,
785 size_t len)
786{
787 int rval;
788 int i = 0;
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100789 unsigned long pageaddr;
790 unsigned long offset;
791 size_t buflen;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700792 int page_count;
793
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100794 pageaddr = (unsigned long)mem & PAGE_CACHE_MASK;
795 offset = (unsigned long)mem - pageaddr;
796 buflen = PAGE_CACHE_ALIGN(len + offset);
797 page_count = buflen >> PAGE_CACHE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700798
799 /* Free any previous set of page pointers */
Nathan Scottce8e9222006-01-11 15:39:08 +1100800 if (bp->b_pages)
801 _xfs_buf_free_pages(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802
Nathan Scottce8e9222006-01-11 15:39:08 +1100803 bp->b_pages = NULL;
804 bp->b_addr = mem;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700805
Christoph Hellwig36fae172009-07-18 18:14:58 -0400806 rval = _xfs_buf_get_pages(bp, page_count, XBF_DONT_BLOCK);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700807 if (rval)
808 return rval;
809
Nathan Scottce8e9222006-01-11 15:39:08 +1100810 bp->b_offset = offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700811
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100812 for (i = 0; i < bp->b_page_count; i++) {
813 bp->b_pages[i] = mem_to_page((void *)pageaddr);
814 pageaddr += PAGE_CACHE_SIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700816
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100817 bp->b_count_desired = len;
818 bp->b_buffer_length = buflen;
Nathan Scottce8e9222006-01-11 15:39:08 +1100819 bp->b_flags |= XBF_MAPPED;
Christoph Hellwig6ab455e2008-05-19 16:34:42 +1000820 bp->b_flags &= ~_XBF_PAGE_LOCKED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821
822 return 0;
823}
824
825xfs_buf_t *
Dave Chinner686865f2010-09-24 20:07:47 +1000826xfs_buf_get_uncached(
827 struct xfs_buftarg *target,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700828 size_t len,
Dave Chinner686865f2010-09-24 20:07:47 +1000829 int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700830{
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000831 unsigned long page_count = PAGE_ALIGN(len) >> PAGE_SHIFT;
832 int error, i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700833 xfs_buf_t *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700834
Nathan Scottce8e9222006-01-11 15:39:08 +1100835 bp = xfs_buf_allocate(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700836 if (unlikely(bp == NULL))
837 goto fail;
Nathan Scottce8e9222006-01-11 15:39:08 +1100838 _xfs_buf_initialize(bp, target, 0, len, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700839
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000840 error = _xfs_buf_get_pages(bp, page_count, 0);
841 if (error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700842 goto fail_free_buf;
843
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000844 for (i = 0; i < page_count; i++) {
Dave Chinner686865f2010-09-24 20:07:47 +1000845 bp->b_pages[i] = alloc_page(xb_to_gfp(flags));
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000846 if (!bp->b_pages[i])
847 goto fail_free_mem;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700848 }
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000849 bp->b_flags |= _XBF_PAGES;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700850
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000851 error = _xfs_buf_map_pages(bp, XBF_MAPPED);
852 if (unlikely(error)) {
853 printk(KERN_WARNING "%s: failed to map pages\n",
Harvey Harrison34a622b2008-04-10 12:19:21 +1000854 __func__);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700855 goto fail_free_mem;
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000856 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700857
Nathan Scottce8e9222006-01-11 15:39:08 +1100858 xfs_buf_unlock(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700859
Dave Chinner686865f2010-09-24 20:07:47 +1000860 trace_xfs_buf_get_uncached(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700861 return bp;
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000862
Linus Torvalds1da177e2005-04-16 15:20:36 -0700863 fail_free_mem:
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000864 while (--i >= 0)
865 __free_page(bp->b_pages[i]);
Christoph Hellwigca165b82007-05-24 15:21:11 +1000866 _xfs_buf_free_pages(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700867 fail_free_buf:
Christoph Hellwigca165b82007-05-24 15:21:11 +1000868 xfs_buf_deallocate(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700869 fail:
870 return NULL;
871}
872
873/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700874 * Increment reference count on buffer, to hold the buffer concurrently
875 * with another thread which may release (free) the buffer asynchronously.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700876 * Must hold the buffer already to call this function.
877 */
878void
Nathan Scottce8e9222006-01-11 15:39:08 +1100879xfs_buf_hold(
880 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700881{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000882 trace_xfs_buf_hold(bp, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +1100883 atomic_inc(&bp->b_hold);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700884}
885
886/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100887 * Releases a hold on the specified buffer. If the
888 * the hold count is 1, calls xfs_buf_free.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889 */
890void
Nathan Scottce8e9222006-01-11 15:39:08 +1100891xfs_buf_rele(
892 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700893{
Dave Chinner74f75a02010-09-24 19:59:04 +1000894 struct xfs_perag *pag = bp->b_pag;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700895
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000896 trace_xfs_buf_rele(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700897
Dave Chinner74f75a02010-09-24 19:59:04 +1000898 if (!pag) {
Nathan Scottfad3aa12006-02-01 12:14:52 +1100899 ASSERT(!bp->b_relse);
Dave Chinner430cbeb2010-12-02 16:30:55 +1100900 ASSERT(list_empty(&bp->b_lru));
Dave Chinner74f75a02010-09-24 19:59:04 +1000901 ASSERT(RB_EMPTY_NODE(&bp->b_rbnode));
Nathan Scottfad3aa12006-02-01 12:14:52 +1100902 if (atomic_dec_and_test(&bp->b_hold))
903 xfs_buf_free(bp);
904 return;
905 }
906
Dave Chinner74f75a02010-09-24 19:59:04 +1000907 ASSERT(!RB_EMPTY_NODE(&bp->b_rbnode));
Dave Chinner430cbeb2010-12-02 16:30:55 +1100908
Lachlan McIlroy37906892008-08-13 15:42:10 +1000909 ASSERT(atomic_read(&bp->b_hold) > 0);
Dave Chinner74f75a02010-09-24 19:59:04 +1000910 if (atomic_dec_and_lock(&bp->b_hold, &pag->pag_buf_lock)) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100911 if (bp->b_relse) {
912 atomic_inc(&bp->b_hold);
Dave Chinner74f75a02010-09-24 19:59:04 +1000913 spin_unlock(&pag->pag_buf_lock);
914 bp->b_relse(bp);
Dave Chinner430cbeb2010-12-02 16:30:55 +1100915 } else if (!(bp->b_flags & XBF_STALE) &&
916 atomic_read(&bp->b_lru_ref)) {
917 xfs_buf_lru_add(bp);
918 spin_unlock(&pag->pag_buf_lock);
Christoph Hellwig7f14d0a2005-11-02 15:09:35 +1100919 } else {
Dave Chinner430cbeb2010-12-02 16:30:55 +1100920 xfs_buf_lru_del(bp);
Nathan Scottce8e9222006-01-11 15:39:08 +1100921 ASSERT(!(bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)));
Dave Chinner74f75a02010-09-24 19:59:04 +1000922 rb_erase(&bp->b_rbnode, &pag->pag_buf_tree);
923 spin_unlock(&pag->pag_buf_lock);
924 xfs_perag_put(pag);
Nathan Scottce8e9222006-01-11 15:39:08 +1100925 xfs_buf_free(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700926 }
927 }
928}
929
930
931/*
932 * Mutual exclusion on buffers. Locking model:
933 *
934 * Buffers associated with inodes for which buffer locking
935 * is not enabled are not protected by semaphores, and are
936 * assumed to be exclusively owned by the caller. There is a
937 * spinlock in the buffer, used by the caller when concurrent
938 * access is possible.
939 */
940
941/*
Dave Chinner90810b92010-11-30 15:16:16 +1100942 * Locks a buffer object, if it is not already locked. Note that this in
943 * no way locks the underlying pages, so it is only useful for
944 * synchronizing concurrent use of buffer objects, not for synchronizing
945 * independent access to the underlying pages.
946 *
947 * If we come across a stale, pinned, locked buffer, we know that we are
948 * being asked to lock a buffer that has been reallocated. Because it is
949 * pinned, we know that the log has not been pushed to disk and hence it
950 * will still be locked. Rather than continuing to have trylock attempts
951 * fail until someone else pushes the log, push it ourselves before
952 * returning. This means that the xfsaild will not get stuck trying
953 * to push on stale inode buffers.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700954 */
955int
Nathan Scottce8e9222006-01-11 15:39:08 +1100956xfs_buf_cond_lock(
957 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958{
959 int locked;
960
Nathan Scottce8e9222006-01-11 15:39:08 +1100961 locked = down_trylock(&bp->b_sema) == 0;
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000962 if (locked)
Nathan Scottce8e9222006-01-11 15:39:08 +1100963 XB_SET_OWNER(bp);
Dave Chinner90810b92010-11-30 15:16:16 +1100964 else if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE))
965 xfs_log_force(bp->b_target->bt_mount, 0);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000966
967 trace_xfs_buf_cond_lock(bp, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +1100968 return locked ? 0 : -EBUSY;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969}
970
Linus Torvalds1da177e2005-04-16 15:20:36 -0700971int
Nathan Scottce8e9222006-01-11 15:39:08 +1100972xfs_buf_lock_value(
973 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700974{
Stephen Rothwelladaa6932008-04-22 15:26:13 +1000975 return bp->b_sema.count;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700976}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700977
978/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100979 * Locks a buffer object.
980 * Note that this in no way locks the underlying pages, so it is only
981 * useful for synchronizing concurrent use of buffer objects, not for
982 * synchronizing independent access to the underlying pages.
Dave Chinnered3b4d62010-05-21 12:07:08 +1000983 *
984 * If we come across a stale, pinned, locked buffer, we know that we
985 * are being asked to lock a buffer that has been reallocated. Because
986 * it is pinned, we know that the log has not been pushed to disk and
987 * hence it will still be locked. Rather than sleeping until someone
988 * else pushes the log, push it ourselves before trying to get the lock.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700989 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100990void
991xfs_buf_lock(
992 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700993{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000994 trace_xfs_buf_lock(bp, _RET_IP_);
995
Dave Chinnered3b4d62010-05-21 12:07:08 +1000996 if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE))
Dave Chinnerebad8612010-09-22 10:47:20 +1000997 xfs_log_force(bp->b_target->bt_mount, 0);
Nathan Scottce8e9222006-01-11 15:39:08 +1100998 if (atomic_read(&bp->b_io_remaining))
999 blk_run_address_space(bp->b_target->bt_mapping);
1000 down(&bp->b_sema);
1001 XB_SET_OWNER(bp);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001002
1003 trace_xfs_buf_lock_done(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001004}
1005
1006/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001007 * Releases the lock on the buffer object.
David Chinner2f926582005-09-05 08:33:35 +10001008 * If the buffer is marked delwri but is not queued, do so before we
Nathan Scottce8e9222006-01-11 15:39:08 +11001009 * unlock the buffer as we need to set flags correctly. We also need to
David Chinner2f926582005-09-05 08:33:35 +10001010 * take a reference for the delwri queue because the unlocker is going to
1011 * drop their's and they don't know we just queued it.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001012 */
1013void
Nathan Scottce8e9222006-01-11 15:39:08 +11001014xfs_buf_unlock(
1015 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001016{
Nathan Scottce8e9222006-01-11 15:39:08 +11001017 if ((bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)) == XBF_DELWRI) {
1018 atomic_inc(&bp->b_hold);
1019 bp->b_flags |= XBF_ASYNC;
1020 xfs_buf_delwri_queue(bp, 0);
David Chinner2f926582005-09-05 08:33:35 +10001021 }
1022
Nathan Scottce8e9222006-01-11 15:39:08 +11001023 XB_CLEAR_OWNER(bp);
1024 up(&bp->b_sema);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001025
1026 trace_xfs_buf_unlock(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001027}
1028
Nathan Scottce8e9222006-01-11 15:39:08 +11001029STATIC void
1030xfs_buf_wait_unpin(
1031 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001032{
1033 DECLARE_WAITQUEUE (wait, current);
1034
Nathan Scottce8e9222006-01-11 15:39:08 +11001035 if (atomic_read(&bp->b_pin_count) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001036 return;
1037
Nathan Scottce8e9222006-01-11 15:39:08 +11001038 add_wait_queue(&bp->b_waiters, &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001039 for (;;) {
1040 set_current_state(TASK_UNINTERRUPTIBLE);
Nathan Scottce8e9222006-01-11 15:39:08 +11001041 if (atomic_read(&bp->b_pin_count) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001042 break;
Nathan Scottce8e9222006-01-11 15:39:08 +11001043 if (atomic_read(&bp->b_io_remaining))
1044 blk_run_address_space(bp->b_target->bt_mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001045 schedule();
1046 }
Nathan Scottce8e9222006-01-11 15:39:08 +11001047 remove_wait_queue(&bp->b_waiters, &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001048 set_current_state(TASK_RUNNING);
1049}
1050
1051/*
1052 * Buffer Utility Routines
1053 */
1054
Linus Torvalds1da177e2005-04-16 15:20:36 -07001055STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001056xfs_buf_iodone_work(
David Howellsc4028952006-11-22 14:57:56 +00001057 struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001058{
David Howellsc4028952006-11-22 14:57:56 +00001059 xfs_buf_t *bp =
1060 container_of(work, xfs_buf_t, b_iodone_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001061
Christoph Hellwig80f6c292010-08-18 05:29:11 -04001062 if (bp->b_iodone)
Nathan Scottce8e9222006-01-11 15:39:08 +11001063 (*(bp->b_iodone))(bp);
1064 else if (bp->b_flags & XBF_ASYNC)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001065 xfs_buf_relse(bp);
1066}
1067
1068void
Nathan Scottce8e9222006-01-11 15:39:08 +11001069xfs_buf_ioend(
1070 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001071 int schedule)
1072{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001073 trace_xfs_buf_iodone(bp, _RET_IP_);
1074
Lachlan McIlroy77be55a2007-11-23 16:31:00 +11001075 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
Nathan Scottce8e9222006-01-11 15:39:08 +11001076 if (bp->b_error == 0)
1077 bp->b_flags |= XBF_DONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001078
Nathan Scottce8e9222006-01-11 15:39:08 +11001079 if ((bp->b_iodone) || (bp->b_flags & XBF_ASYNC)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001080 if (schedule) {
David Howellsc4028952006-11-22 14:57:56 +00001081 INIT_WORK(&bp->b_iodone_work, xfs_buf_iodone_work);
Nathan Scottce8e9222006-01-11 15:39:08 +11001082 queue_work(xfslogd_workqueue, &bp->b_iodone_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001083 } else {
David Howellsc4028952006-11-22 14:57:56 +00001084 xfs_buf_iodone_work(&bp->b_iodone_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001085 }
1086 } else {
David Chinnerb4dd3302008-08-13 16:36:11 +10001087 complete(&bp->b_iowait);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001088 }
1089}
1090
Linus Torvalds1da177e2005-04-16 15:20:36 -07001091void
Nathan Scottce8e9222006-01-11 15:39:08 +11001092xfs_buf_ioerror(
1093 xfs_buf_t *bp,
1094 int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001095{
1096 ASSERT(error >= 0 && error <= 0xffff);
Nathan Scottce8e9222006-01-11 15:39:08 +11001097 bp->b_error = (unsigned short)error;
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001098 trace_xfs_buf_ioerror(bp, error, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001099}
1100
Linus Torvalds1da177e2005-04-16 15:20:36 -07001101int
Christoph Hellwig64e0bc72010-01-13 22:17:58 +00001102xfs_bwrite(
1103 struct xfs_mount *mp,
Christoph Hellwig5d765b92008-12-03 12:20:26 +01001104 struct xfs_buf *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001105{
Christoph Hellwig8c383662010-03-12 10:59:40 +00001106 int error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001107
Christoph Hellwig64e0bc72010-01-13 22:17:58 +00001108 bp->b_flags |= XBF_WRITE;
Christoph Hellwig8c383662010-03-12 10:59:40 +00001109 bp->b_flags &= ~(XBF_ASYNC | XBF_READ);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001110
Christoph Hellwig5d765b92008-12-03 12:20:26 +01001111 xfs_buf_delwri_dequeue(bp);
Christoph Hellwig939d7232010-07-20 17:51:16 +10001112 xfs_bdstrat_cb(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001113
Christoph Hellwig8c383662010-03-12 10:59:40 +00001114 error = xfs_buf_iowait(bp);
1115 if (error)
1116 xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
1117 xfs_buf_relse(bp);
Christoph Hellwig64e0bc72010-01-13 22:17:58 +00001118 return error;
Christoph Hellwig5d765b92008-12-03 12:20:26 +01001119}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001120
Christoph Hellwig5d765b92008-12-03 12:20:26 +01001121void
1122xfs_bdwrite(
1123 void *mp,
1124 struct xfs_buf *bp)
1125{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001126 trace_xfs_buf_bdwrite(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001127
Christoph Hellwig5d765b92008-12-03 12:20:26 +01001128 bp->b_flags &= ~XBF_READ;
1129 bp->b_flags |= (XBF_DELWRI | XBF_ASYNC);
1130
1131 xfs_buf_delwri_queue(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001132}
1133
Christoph Hellwig4e234712010-01-13 22:17:56 +00001134/*
1135 * Called when we want to stop a buffer from getting written or read.
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +00001136 * We attach the EIO error, muck with its flags, and call xfs_buf_ioend
Christoph Hellwig4e234712010-01-13 22:17:56 +00001137 * so that the proper iodone callbacks get called.
1138 */
1139STATIC int
1140xfs_bioerror(
1141 xfs_buf_t *bp)
1142{
1143#ifdef XFSERRORDEBUG
1144 ASSERT(XFS_BUF_ISREAD(bp) || bp->b_iodone);
1145#endif
1146
1147 /*
1148 * No need to wait until the buffer is unpinned, we aren't flushing it.
1149 */
1150 XFS_BUF_ERROR(bp, EIO);
1151
1152 /*
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +00001153 * We're calling xfs_buf_ioend, so delete XBF_DONE flag.
Christoph Hellwig4e234712010-01-13 22:17:56 +00001154 */
1155 XFS_BUF_UNREAD(bp);
1156 XFS_BUF_UNDELAYWRITE(bp);
1157 XFS_BUF_UNDONE(bp);
1158 XFS_BUF_STALE(bp);
1159
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +00001160 xfs_buf_ioend(bp, 0);
Christoph Hellwig4e234712010-01-13 22:17:56 +00001161
1162 return EIO;
1163}
1164
1165/*
1166 * Same as xfs_bioerror, except that we are releasing the buffer
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +00001167 * here ourselves, and avoiding the xfs_buf_ioend call.
Christoph Hellwig4e234712010-01-13 22:17:56 +00001168 * This is meant for userdata errors; metadata bufs come with
1169 * iodone functions attached, so that we can track down errors.
1170 */
1171STATIC int
1172xfs_bioerror_relse(
1173 struct xfs_buf *bp)
1174{
1175 int64_t fl = XFS_BUF_BFLAGS(bp);
1176 /*
1177 * No need to wait until the buffer is unpinned.
1178 * We aren't flushing it.
1179 *
1180 * chunkhold expects B_DONE to be set, whether
1181 * we actually finish the I/O or not. We don't want to
1182 * change that interface.
1183 */
1184 XFS_BUF_UNREAD(bp);
1185 XFS_BUF_UNDELAYWRITE(bp);
1186 XFS_BUF_DONE(bp);
1187 XFS_BUF_STALE(bp);
1188 XFS_BUF_CLR_IODONE_FUNC(bp);
Christoph Hellwig0cadda12010-01-19 09:56:44 +00001189 if (!(fl & XBF_ASYNC)) {
Christoph Hellwig4e234712010-01-13 22:17:56 +00001190 /*
1191 * Mark b_error and B_ERROR _both_.
1192 * Lot's of chunkcache code assumes that.
1193 * There's no reason to mark error for
1194 * ASYNC buffers.
1195 */
1196 XFS_BUF_ERROR(bp, EIO);
1197 XFS_BUF_FINISH_IOWAIT(bp);
1198 } else {
1199 xfs_buf_relse(bp);
1200 }
1201
1202 return EIO;
1203}
1204
1205
1206/*
1207 * All xfs metadata buffers except log state machine buffers
1208 * get this attached as their b_bdstrat callback function.
1209 * This is so that we can catch a buffer
1210 * after prematurely unpinning it to forcibly shutdown the filesystem.
1211 */
1212int
1213xfs_bdstrat_cb(
1214 struct xfs_buf *bp)
1215{
Dave Chinnerebad8612010-09-22 10:47:20 +10001216 if (XFS_FORCED_SHUTDOWN(bp->b_target->bt_mount)) {
Christoph Hellwig4e234712010-01-13 22:17:56 +00001217 trace_xfs_bdstrat_shut(bp, _RET_IP_);
1218 /*
1219 * Metadata write that didn't get logged but
1220 * written delayed anyway. These aren't associated
1221 * with a transaction, and can be ignored.
1222 */
1223 if (!bp->b_iodone && !XFS_BUF_ISREAD(bp))
1224 return xfs_bioerror_relse(bp);
1225 else
1226 return xfs_bioerror(bp);
1227 }
1228
1229 xfs_buf_iorequest(bp);
1230 return 0;
1231}
1232
1233/*
1234 * Wrapper around bdstrat so that we can stop data from going to disk in case
1235 * we are shutting down the filesystem. Typically user data goes thru this
1236 * path; one of the exceptions is the superblock.
1237 */
1238void
1239xfsbdstrat(
1240 struct xfs_mount *mp,
1241 struct xfs_buf *bp)
1242{
1243 if (XFS_FORCED_SHUTDOWN(mp)) {
1244 trace_xfs_bdstrat_shut(bp, _RET_IP_);
1245 xfs_bioerror_relse(bp);
1246 return;
1247 }
1248
1249 xfs_buf_iorequest(bp);
1250}
1251
Christoph Hellwigb8f82a42009-11-14 16:17:22 +00001252STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001253_xfs_buf_ioend(
1254 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001255 int schedule)
1256{
Christoph Hellwig6ab455e2008-05-19 16:34:42 +10001257 if (atomic_dec_and_test(&bp->b_io_remaining) == 1) {
1258 bp->b_flags &= ~_XBF_PAGE_LOCKED;
Nathan Scottce8e9222006-01-11 15:39:08 +11001259 xfs_buf_ioend(bp, schedule);
Christoph Hellwig6ab455e2008-05-19 16:34:42 +10001260 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001261}
1262
Al Viro782e3b32007-10-12 07:17:47 +01001263STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001264xfs_buf_bio_end_io(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001265 struct bio *bio,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001266 int error)
1267{
Nathan Scottce8e9222006-01-11 15:39:08 +11001268 xfs_buf_t *bp = (xfs_buf_t *)bio->bi_private;
1269 unsigned int blocksize = bp->b_target->bt_bsize;
Nathan Scotteedb5532005-09-02 16:39:56 +10001270 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001271
Lachlan McIlroycfbe5262008-12-12 15:27:25 +11001272 xfs_buf_ioerror(bp, -error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001273
James Bottomley73c77e22010-01-25 11:42:24 -06001274 if (!error && xfs_buf_is_vmapped(bp) && (bp->b_flags & XBF_READ))
1275 invalidate_kernel_vmap_range(bp->b_addr, xfs_buf_vmap_len(bp));
1276
Nathan Scotteedb5532005-09-02 16:39:56 +10001277 do {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001278 struct page *page = bvec->bv_page;
1279
Nathan Scott948ecdb2006-09-28 11:03:13 +10001280 ASSERT(!PagePrivate(page));
Nathan Scottce8e9222006-01-11 15:39:08 +11001281 if (unlikely(bp->b_error)) {
1282 if (bp->b_flags & XBF_READ)
Nathan Scotteedb5532005-09-02 16:39:56 +10001283 ClearPageUptodate(page);
Nathan Scottce8e9222006-01-11 15:39:08 +11001284 } else if (blocksize >= PAGE_CACHE_SIZE) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001285 SetPageUptodate(page);
1286 } else if (!PagePrivate(page) &&
Nathan Scottce8e9222006-01-11 15:39:08 +11001287 (bp->b_flags & _XBF_PAGE_CACHE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001288 set_page_region(page, bvec->bv_offset, bvec->bv_len);
1289 }
1290
Nathan Scotteedb5532005-09-02 16:39:56 +10001291 if (--bvec >= bio->bi_io_vec)
1292 prefetchw(&bvec->bv_page->flags);
Christoph Hellwig6ab455e2008-05-19 16:34:42 +10001293
1294 if (bp->b_flags & _XBF_PAGE_LOCKED)
1295 unlock_page(page);
Nathan Scotteedb5532005-09-02 16:39:56 +10001296 } while (bvec >= bio->bi_io_vec);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001297
Nathan Scottce8e9222006-01-11 15:39:08 +11001298 _xfs_buf_ioend(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001299 bio_put(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001300}
1301
1302STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001303_xfs_buf_ioapply(
1304 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001305{
Christoph Hellwiga9759f22007-12-07 14:07:08 +11001306 int rw, map_i, total_nr_pages, nr_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001307 struct bio *bio;
Nathan Scottce8e9222006-01-11 15:39:08 +11001308 int offset = bp->b_offset;
1309 int size = bp->b_count_desired;
1310 sector_t sector = bp->b_bn;
1311 unsigned int blocksize = bp->b_target->bt_bsize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001312
Nathan Scottce8e9222006-01-11 15:39:08 +11001313 total_nr_pages = bp->b_page_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001314 map_i = 0;
1315
Nathan Scottce8e9222006-01-11 15:39:08 +11001316 if (bp->b_flags & XBF_ORDERED) {
1317 ASSERT(!(bp->b_flags & XBF_READ));
Christoph Hellwig80f6c292010-08-18 05:29:11 -04001318 rw = WRITE_FLUSH_FUA;
Dave Chinner2ee1aba2009-11-24 18:03:15 +00001319 } else if (bp->b_flags & XBF_LOG_BUFFER) {
Nathan Scott51bdd702006-09-28 11:01:57 +10001320 ASSERT(!(bp->b_flags & XBF_READ_AHEAD));
1321 bp->b_flags &= ~_XBF_RUN_QUEUES;
1322 rw = (bp->b_flags & XBF_WRITE) ? WRITE_SYNC : READ_SYNC;
Dave Chinner2ee1aba2009-11-24 18:03:15 +00001323 } else if (bp->b_flags & _XBF_RUN_QUEUES) {
1324 ASSERT(!(bp->b_flags & XBF_READ_AHEAD));
1325 bp->b_flags &= ~_XBF_RUN_QUEUES;
1326 rw = (bp->b_flags & XBF_WRITE) ? WRITE_META : READ_META;
Nathan Scott51bdd702006-09-28 11:01:57 +10001327 } else {
1328 rw = (bp->b_flags & XBF_WRITE) ? WRITE :
1329 (bp->b_flags & XBF_READ_AHEAD) ? READA : READ;
Christoph Hellwigf538d4d2005-11-02 10:26:59 +11001330 }
1331
Nathan Scottce8e9222006-01-11 15:39:08 +11001332 /* Special code path for reading a sub page size buffer in --
Linus Torvalds1da177e2005-04-16 15:20:36 -07001333 * we populate up the whole page, and hence the other metadata
1334 * in the same page. This optimization is only valid when the
Nathan Scottce8e9222006-01-11 15:39:08 +11001335 * filesystem block size is not smaller than the page size.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001336 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001337 if ((bp->b_buffer_length < PAGE_CACHE_SIZE) &&
Christoph Hellwig6ab455e2008-05-19 16:34:42 +10001338 ((bp->b_flags & (XBF_READ|_XBF_PAGE_LOCKED)) ==
1339 (XBF_READ|_XBF_PAGE_LOCKED)) &&
Nathan Scottce8e9222006-01-11 15:39:08 +11001340 (blocksize >= PAGE_CACHE_SIZE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001341 bio = bio_alloc(GFP_NOIO, 1);
1342
Nathan Scottce8e9222006-01-11 15:39:08 +11001343 bio->bi_bdev = bp->b_target->bt_bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001344 bio->bi_sector = sector - (offset >> BBSHIFT);
Nathan Scottce8e9222006-01-11 15:39:08 +11001345 bio->bi_end_io = xfs_buf_bio_end_io;
1346 bio->bi_private = bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001347
Nathan Scottce8e9222006-01-11 15:39:08 +11001348 bio_add_page(bio, bp->b_pages[0], PAGE_CACHE_SIZE, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001349 size = 0;
1350
Nathan Scottce8e9222006-01-11 15:39:08 +11001351 atomic_inc(&bp->b_io_remaining);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001352
1353 goto submit_io;
1354 }
1355
Linus Torvalds1da177e2005-04-16 15:20:36 -07001356next_chunk:
Nathan Scottce8e9222006-01-11 15:39:08 +11001357 atomic_inc(&bp->b_io_remaining);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001358 nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1359 if (nr_pages > total_nr_pages)
1360 nr_pages = total_nr_pages;
1361
1362 bio = bio_alloc(GFP_NOIO, nr_pages);
Nathan Scottce8e9222006-01-11 15:39:08 +11001363 bio->bi_bdev = bp->b_target->bt_bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001364 bio->bi_sector = sector;
Nathan Scottce8e9222006-01-11 15:39:08 +11001365 bio->bi_end_io = xfs_buf_bio_end_io;
1366 bio->bi_private = bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001367
1368 for (; size && nr_pages; nr_pages--, map_i++) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001369 int rbytes, nbytes = PAGE_CACHE_SIZE - offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001370
1371 if (nbytes > size)
1372 nbytes = size;
1373
Nathan Scottce8e9222006-01-11 15:39:08 +11001374 rbytes = bio_add_page(bio, bp->b_pages[map_i], nbytes, offset);
1375 if (rbytes < nbytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001376 break;
1377
1378 offset = 0;
1379 sector += nbytes >> BBSHIFT;
1380 size -= nbytes;
1381 total_nr_pages--;
1382 }
1383
1384submit_io:
1385 if (likely(bio->bi_size)) {
James Bottomley73c77e22010-01-25 11:42:24 -06001386 if (xfs_buf_is_vmapped(bp)) {
1387 flush_kernel_vmap_range(bp->b_addr,
1388 xfs_buf_vmap_len(bp));
1389 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001390 submit_bio(rw, bio);
1391 if (size)
1392 goto next_chunk;
1393 } else {
Dave Chinnerec53d1d2010-07-20 17:52:59 +10001394 /*
1395 * if we get here, no pages were added to the bio. However,
1396 * we can't just error out here - if the pages are locked then
1397 * we have to unlock them otherwise we can hang on a later
1398 * access to the page.
1399 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001400 xfs_buf_ioerror(bp, EIO);
Dave Chinnerec53d1d2010-07-20 17:52:59 +10001401 if (bp->b_flags & _XBF_PAGE_LOCKED) {
1402 int i;
1403 for (i = 0; i < bp->b_page_count; i++)
1404 unlock_page(bp->b_pages[i]);
1405 }
1406 bio_put(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001407 }
1408}
1409
Linus Torvalds1da177e2005-04-16 15:20:36 -07001410int
Nathan Scottce8e9222006-01-11 15:39:08 +11001411xfs_buf_iorequest(
1412 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001413{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001414 trace_xfs_buf_iorequest(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001415
Nathan Scottce8e9222006-01-11 15:39:08 +11001416 if (bp->b_flags & XBF_DELWRI) {
1417 xfs_buf_delwri_queue(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001418 return 0;
1419 }
1420
Nathan Scottce8e9222006-01-11 15:39:08 +11001421 if (bp->b_flags & XBF_WRITE) {
1422 xfs_buf_wait_unpin(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001423 }
1424
Nathan Scottce8e9222006-01-11 15:39:08 +11001425 xfs_buf_hold(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001426
1427 /* Set the count to 1 initially, this will stop an I/O
1428 * completion callout which happens before we have started
Nathan Scottce8e9222006-01-11 15:39:08 +11001429 * all the I/O from calling xfs_buf_ioend too early.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001430 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001431 atomic_set(&bp->b_io_remaining, 1);
1432 _xfs_buf_ioapply(bp);
1433 _xfs_buf_ioend(bp, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001434
Nathan Scottce8e9222006-01-11 15:39:08 +11001435 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001436 return 0;
1437}
1438
1439/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001440 * Waits for I/O to complete on the buffer supplied.
1441 * It returns immediately if no I/O is pending.
1442 * It returns the I/O error code, if any, or 0 if there was no error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001443 */
1444int
Nathan Scottce8e9222006-01-11 15:39:08 +11001445xfs_buf_iowait(
1446 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001447{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001448 trace_xfs_buf_iowait(bp, _RET_IP_);
1449
Nathan Scottce8e9222006-01-11 15:39:08 +11001450 if (atomic_read(&bp->b_io_remaining))
1451 blk_run_address_space(bp->b_target->bt_mapping);
David Chinnerb4dd3302008-08-13 16:36:11 +10001452 wait_for_completion(&bp->b_iowait);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001453
1454 trace_xfs_buf_iowait_done(bp, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +11001455 return bp->b_error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001456}
1457
Nathan Scottce8e9222006-01-11 15:39:08 +11001458xfs_caddr_t
1459xfs_buf_offset(
1460 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001461 size_t offset)
1462{
1463 struct page *page;
1464
Nathan Scottce8e9222006-01-11 15:39:08 +11001465 if (bp->b_flags & XBF_MAPPED)
1466 return XFS_BUF_PTR(bp) + offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001467
Nathan Scottce8e9222006-01-11 15:39:08 +11001468 offset += bp->b_offset;
1469 page = bp->b_pages[offset >> PAGE_CACHE_SHIFT];
1470 return (xfs_caddr_t)page_address(page) + (offset & (PAGE_CACHE_SIZE-1));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001471}
1472
1473/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001474 * Move data into or out of a buffer.
1475 */
1476void
Nathan Scottce8e9222006-01-11 15:39:08 +11001477xfs_buf_iomove(
1478 xfs_buf_t *bp, /* buffer to process */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001479 size_t boff, /* starting buffer offset */
1480 size_t bsize, /* length to copy */
Dave Chinnerb9c48642010-01-20 10:47:39 +11001481 void *data, /* data address */
Nathan Scottce8e9222006-01-11 15:39:08 +11001482 xfs_buf_rw_t mode) /* read/write/zero flag */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001483{
1484 size_t bend, cpoff, csize;
1485 struct page *page;
1486
1487 bend = boff + bsize;
1488 while (boff < bend) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001489 page = bp->b_pages[xfs_buf_btoct(boff + bp->b_offset)];
1490 cpoff = xfs_buf_poff(boff + bp->b_offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001491 csize = min_t(size_t,
Nathan Scottce8e9222006-01-11 15:39:08 +11001492 PAGE_CACHE_SIZE-cpoff, bp->b_count_desired-boff);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001493
1494 ASSERT(((csize + cpoff) <= PAGE_CACHE_SIZE));
1495
1496 switch (mode) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001497 case XBRW_ZERO:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001498 memset(page_address(page) + cpoff, 0, csize);
1499 break;
Nathan Scottce8e9222006-01-11 15:39:08 +11001500 case XBRW_READ:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001501 memcpy(data, page_address(page) + cpoff, csize);
1502 break;
Nathan Scottce8e9222006-01-11 15:39:08 +11001503 case XBRW_WRITE:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001504 memcpy(page_address(page) + cpoff, data, csize);
1505 }
1506
1507 boff += csize;
1508 data += csize;
1509 }
1510}
1511
1512/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001513 * Handling of buffer targets (buftargs).
Linus Torvalds1da177e2005-04-16 15:20:36 -07001514 */
1515
1516/*
Dave Chinner430cbeb2010-12-02 16:30:55 +11001517 * Wait for any bufs with callbacks that have been submitted but have not yet
1518 * returned. These buffers will have an elevated hold count, so wait on those
1519 * while freeing all the buffers only held by the LRU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001520 */
1521void
1522xfs_wait_buftarg(
Dave Chinner74f75a02010-09-24 19:59:04 +10001523 struct xfs_buftarg *btp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001524{
Dave Chinner430cbeb2010-12-02 16:30:55 +11001525 struct xfs_buf *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001526
Dave Chinner430cbeb2010-12-02 16:30:55 +11001527restart:
1528 spin_lock(&btp->bt_lru_lock);
1529 while (!list_empty(&btp->bt_lru)) {
1530 bp = list_first_entry(&btp->bt_lru, struct xfs_buf, b_lru);
1531 if (atomic_read(&bp->b_hold) > 1) {
1532 spin_unlock(&btp->bt_lru_lock);
Dave Chinner26af6552010-09-22 10:47:20 +10001533 delay(100);
Dave Chinner430cbeb2010-12-02 16:30:55 +11001534 goto restart;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001535 }
Dave Chinner430cbeb2010-12-02 16:30:55 +11001536 /*
1537 * clear the LRU reference count so the bufer doesn't get
1538 * ignored in xfs_buf_rele().
1539 */
1540 atomic_set(&bp->b_lru_ref, 0);
1541 spin_unlock(&btp->bt_lru_lock);
1542 xfs_buf_rele(bp);
1543 spin_lock(&btp->bt_lru_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001544 }
Dave Chinner430cbeb2010-12-02 16:30:55 +11001545 spin_unlock(&btp->bt_lru_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001546}
1547
Dave Chinnerff57ab22010-11-30 17:27:57 +11001548int
1549xfs_buftarg_shrink(
1550 struct shrinker *shrink,
1551 int nr_to_scan,
1552 gfp_t mask)
David Chinnera6867a62006-01-11 15:37:58 +11001553{
Dave Chinnerff57ab22010-11-30 17:27:57 +11001554 struct xfs_buftarg *btp = container_of(shrink,
1555 struct xfs_buftarg, bt_shrinker);
Dave Chinner430cbeb2010-12-02 16:30:55 +11001556 struct xfs_buf *bp;
1557 LIST_HEAD(dispose);
1558
1559 if (!nr_to_scan)
1560 return btp->bt_lru_nr;
1561
1562 spin_lock(&btp->bt_lru_lock);
1563 while (!list_empty(&btp->bt_lru)) {
1564 if (nr_to_scan-- <= 0)
1565 break;
1566
1567 bp = list_first_entry(&btp->bt_lru, struct xfs_buf, b_lru);
1568
1569 /*
1570 * Decrement the b_lru_ref count unless the value is already
1571 * zero. If the value is already zero, we need to reclaim the
1572 * buffer, otherwise it gets another trip through the LRU.
1573 */
1574 if (!atomic_add_unless(&bp->b_lru_ref, -1, 0)) {
1575 list_move_tail(&bp->b_lru, &btp->bt_lru);
1576 continue;
1577 }
1578
1579 /*
1580 * remove the buffer from the LRU now to avoid needing another
1581 * lock round trip inside xfs_buf_rele().
1582 */
1583 list_move(&bp->b_lru, &dispose);
1584 btp->bt_lru_nr--;
Dave Chinnerff57ab22010-11-30 17:27:57 +11001585 }
Dave Chinner430cbeb2010-12-02 16:30:55 +11001586 spin_unlock(&btp->bt_lru_lock);
1587
1588 while (!list_empty(&dispose)) {
1589 bp = list_first_entry(&dispose, struct xfs_buf, b_lru);
1590 list_del_init(&bp->b_lru);
1591 xfs_buf_rele(bp);
1592 }
1593
1594 return btp->bt_lru_nr;
David Chinnera6867a62006-01-11 15:37:58 +11001595}
1596
Linus Torvalds1da177e2005-04-16 15:20:36 -07001597void
1598xfs_free_buftarg(
Christoph Hellwigb7963132009-03-03 14:48:37 -05001599 struct xfs_mount *mp,
1600 struct xfs_buftarg *btp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001601{
Dave Chinnerff57ab22010-11-30 17:27:57 +11001602 unregister_shrinker(&btp->bt_shrinker);
1603
Linus Torvalds1da177e2005-04-16 15:20:36 -07001604 xfs_flush_buftarg(btp, 1);
Christoph Hellwigb7963132009-03-03 14:48:37 -05001605 if (mp->m_flags & XFS_MOUNT_BARRIER)
1606 xfs_blkdev_issue_flush(btp);
Nathan Scottce8e9222006-01-11 15:39:08 +11001607 iput(btp->bt_mapping->host);
David Chinnera6867a62006-01-11 15:37:58 +11001608
David Chinnera6867a62006-01-11 15:37:58 +11001609 kthread_stop(btp->bt_task);
Denys Vlasenkof0e2d932008-05-19 16:31:57 +10001610 kmem_free(btp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001611}
1612
Linus Torvalds1da177e2005-04-16 15:20:36 -07001613STATIC int
1614xfs_setsize_buftarg_flags(
1615 xfs_buftarg_t *btp,
1616 unsigned int blocksize,
1617 unsigned int sectorsize,
1618 int verbose)
1619{
Nathan Scottce8e9222006-01-11 15:39:08 +11001620 btp->bt_bsize = blocksize;
1621 btp->bt_sshift = ffs(sectorsize) - 1;
1622 btp->bt_smask = sectorsize - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001623
Nathan Scottce8e9222006-01-11 15:39:08 +11001624 if (set_blocksize(btp->bt_bdev, sectorsize)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001625 printk(KERN_WARNING
1626 "XFS: Cannot set_blocksize to %u on device %s\n",
1627 sectorsize, XFS_BUFTARG_NAME(btp));
1628 return EINVAL;
1629 }
1630
1631 if (verbose &&
1632 (PAGE_CACHE_SIZE / BITS_PER_LONG) > sectorsize) {
1633 printk(KERN_WARNING
1634 "XFS: %u byte sectors in use on device %s. "
1635 "This is suboptimal; %u or greater is ideal.\n",
1636 sectorsize, XFS_BUFTARG_NAME(btp),
1637 (unsigned int)PAGE_CACHE_SIZE / BITS_PER_LONG);
1638 }
1639
1640 return 0;
1641}
1642
1643/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001644 * When allocating the initial buffer target we have not yet
1645 * read in the superblock, so don't know what sized sectors
1646 * are being used is at this early stage. Play safe.
1647 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001648STATIC int
1649xfs_setsize_buftarg_early(
1650 xfs_buftarg_t *btp,
1651 struct block_device *bdev)
1652{
1653 return xfs_setsize_buftarg_flags(btp,
Martin K. Petersene1defc42009-05-22 17:17:49 -04001654 PAGE_CACHE_SIZE, bdev_logical_block_size(bdev), 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001655}
1656
1657int
1658xfs_setsize_buftarg(
1659 xfs_buftarg_t *btp,
1660 unsigned int blocksize,
1661 unsigned int sectorsize)
1662{
1663 return xfs_setsize_buftarg_flags(btp, blocksize, sectorsize, 1);
1664}
1665
1666STATIC int
1667xfs_mapping_buftarg(
1668 xfs_buftarg_t *btp,
1669 struct block_device *bdev)
1670{
1671 struct backing_dev_info *bdi;
1672 struct inode *inode;
1673 struct address_space *mapping;
Christoph Hellwigf5e54d62006-06-28 04:26:44 -07001674 static const struct address_space_operations mapping_aops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001675 .sync_page = block_sync_page,
Christoph Lametere965f962006-02-01 03:05:41 -08001676 .migratepage = fail_migrate_page,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001677 };
1678
1679 inode = new_inode(bdev->bd_inode->i_sb);
1680 if (!inode) {
1681 printk(KERN_WARNING
1682 "XFS: Cannot allocate mapping inode for device %s\n",
1683 XFS_BUFTARG_NAME(btp));
1684 return ENOMEM;
1685 }
Christoph Hellwig85fe4022010-10-23 11:19:54 -04001686 inode->i_ino = get_next_ino();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001687 inode->i_mode = S_IFBLK;
1688 inode->i_bdev = bdev;
1689 inode->i_rdev = bdev->bd_dev;
1690 bdi = blk_get_backing_dev_info(bdev);
1691 if (!bdi)
1692 bdi = &default_backing_dev_info;
1693 mapping = &inode->i_data;
1694 mapping->a_ops = &mapping_aops;
1695 mapping->backing_dev_info = bdi;
1696 mapping_set_gfp_mask(mapping, GFP_NOFS);
Nathan Scottce8e9222006-01-11 15:39:08 +11001697 btp->bt_mapping = mapping;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001698 return 0;
1699}
1700
David Chinnera6867a62006-01-11 15:37:58 +11001701STATIC int
1702xfs_alloc_delwrite_queue(
Jan Engelhardte2a07812010-03-23 09:52:55 +11001703 xfs_buftarg_t *btp,
1704 const char *fsname)
David Chinnera6867a62006-01-11 15:37:58 +11001705{
David Chinnera6867a62006-01-11 15:37:58 +11001706 INIT_LIST_HEAD(&btp->bt_delwrite_queue);
Eric Sandeen007c61c2007-10-11 17:43:56 +10001707 spin_lock_init(&btp->bt_delwrite_lock);
David Chinnera6867a62006-01-11 15:37:58 +11001708 btp->bt_flags = 0;
Jan Engelhardte2a07812010-03-23 09:52:55 +11001709 btp->bt_task = kthread_run(xfsbufd, btp, "xfsbufd/%s", fsname);
Dave Chinnerff57ab22010-11-30 17:27:57 +11001710 if (IS_ERR(btp->bt_task))
1711 return PTR_ERR(btp->bt_task);
1712 return 0;
David Chinnera6867a62006-01-11 15:37:58 +11001713}
1714
Linus Torvalds1da177e2005-04-16 15:20:36 -07001715xfs_buftarg_t *
1716xfs_alloc_buftarg(
Dave Chinnerebad8612010-09-22 10:47:20 +10001717 struct xfs_mount *mp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001718 struct block_device *bdev,
Jan Engelhardte2a07812010-03-23 09:52:55 +11001719 int external,
1720 const char *fsname)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001721{
1722 xfs_buftarg_t *btp;
1723
1724 btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
1725
Dave Chinnerebad8612010-09-22 10:47:20 +10001726 btp->bt_mount = mp;
Nathan Scottce8e9222006-01-11 15:39:08 +11001727 btp->bt_dev = bdev->bd_dev;
1728 btp->bt_bdev = bdev;
Dave Chinner430cbeb2010-12-02 16:30:55 +11001729 INIT_LIST_HEAD(&btp->bt_lru);
1730 spin_lock_init(&btp->bt_lru_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001731 if (xfs_setsize_buftarg_early(btp, bdev))
1732 goto error;
1733 if (xfs_mapping_buftarg(btp, bdev))
1734 goto error;
Jan Engelhardte2a07812010-03-23 09:52:55 +11001735 if (xfs_alloc_delwrite_queue(btp, fsname))
David Chinnera6867a62006-01-11 15:37:58 +11001736 goto error;
Dave Chinnerff57ab22010-11-30 17:27:57 +11001737 btp->bt_shrinker.shrink = xfs_buftarg_shrink;
1738 btp->bt_shrinker.seeks = DEFAULT_SEEKS;
1739 register_shrinker(&btp->bt_shrinker);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001740 return btp;
1741
1742error:
Denys Vlasenkof0e2d932008-05-19 16:31:57 +10001743 kmem_free(btp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001744 return NULL;
1745}
1746
1747
1748/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001749 * Delayed write buffer handling
Linus Torvalds1da177e2005-04-16 15:20:36 -07001750 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001751STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001752xfs_buf_delwri_queue(
1753 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001754 int unlock)
1755{
Nathan Scottce8e9222006-01-11 15:39:08 +11001756 struct list_head *dwq = &bp->b_target->bt_delwrite_queue;
1757 spinlock_t *dwlk = &bp->b_target->bt_delwrite_lock;
David Chinnera6867a62006-01-11 15:37:58 +11001758
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001759 trace_xfs_buf_delwri_queue(bp, _RET_IP_);
1760
Nathan Scottce8e9222006-01-11 15:39:08 +11001761 ASSERT((bp->b_flags&(XBF_DELWRI|XBF_ASYNC)) == (XBF_DELWRI|XBF_ASYNC));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001762
David Chinnera6867a62006-01-11 15:37:58 +11001763 spin_lock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001764 /* If already in the queue, dequeue and place at tail */
Nathan Scottce8e9222006-01-11 15:39:08 +11001765 if (!list_empty(&bp->b_list)) {
1766 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1767 if (unlock)
1768 atomic_dec(&bp->b_hold);
1769 list_del(&bp->b_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001770 }
1771
Dave Chinnerc9c12972010-01-11 11:49:59 +00001772 if (list_empty(dwq)) {
1773 /* start xfsbufd as it is about to have something to do */
1774 wake_up_process(bp->b_target->bt_task);
1775 }
1776
Nathan Scottce8e9222006-01-11 15:39:08 +11001777 bp->b_flags |= _XBF_DELWRI_Q;
1778 list_add_tail(&bp->b_list, dwq);
1779 bp->b_queuetime = jiffies;
David Chinnera6867a62006-01-11 15:37:58 +11001780 spin_unlock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001781
1782 if (unlock)
Nathan Scottce8e9222006-01-11 15:39:08 +11001783 xfs_buf_unlock(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001784}
1785
1786void
Nathan Scottce8e9222006-01-11 15:39:08 +11001787xfs_buf_delwri_dequeue(
1788 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001789{
Nathan Scottce8e9222006-01-11 15:39:08 +11001790 spinlock_t *dwlk = &bp->b_target->bt_delwrite_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001791 int dequeued = 0;
1792
David Chinnera6867a62006-01-11 15:37:58 +11001793 spin_lock(dwlk);
Nathan Scottce8e9222006-01-11 15:39:08 +11001794 if ((bp->b_flags & XBF_DELWRI) && !list_empty(&bp->b_list)) {
1795 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1796 list_del_init(&bp->b_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001797 dequeued = 1;
1798 }
Nathan Scottce8e9222006-01-11 15:39:08 +11001799 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
David Chinnera6867a62006-01-11 15:37:58 +11001800 spin_unlock(dwlk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001801
1802 if (dequeued)
Nathan Scottce8e9222006-01-11 15:39:08 +11001803 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001804
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001805 trace_xfs_buf_delwri_dequeue(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001806}
1807
Dave Chinnerd808f612010-02-02 10:13:42 +11001808/*
1809 * If a delwri buffer needs to be pushed before it has aged out, then promote
1810 * it to the head of the delwri queue so that it will be flushed on the next
1811 * xfsbufd run. We do this by resetting the queuetime of the buffer to be older
1812 * than the age currently needed to flush the buffer. Hence the next time the
1813 * xfsbufd sees it is guaranteed to be considered old enough to flush.
1814 */
1815void
1816xfs_buf_delwri_promote(
1817 struct xfs_buf *bp)
1818{
1819 struct xfs_buftarg *btp = bp->b_target;
1820 long age = xfs_buf_age_centisecs * msecs_to_jiffies(10) + 1;
1821
1822 ASSERT(bp->b_flags & XBF_DELWRI);
1823 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1824
1825 /*
1826 * Check the buffer age before locking the delayed write queue as we
1827 * don't need to promote buffers that are already past the flush age.
1828 */
1829 if (bp->b_queuetime < jiffies - age)
1830 return;
1831 bp->b_queuetime = jiffies - age;
1832 spin_lock(&btp->bt_delwrite_lock);
1833 list_move(&bp->b_list, &btp->bt_delwrite_queue);
1834 spin_unlock(&btp->bt_delwrite_lock);
1835}
1836
Linus Torvalds1da177e2005-04-16 15:20:36 -07001837STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001838xfs_buf_runall_queues(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001839 struct workqueue_struct *queue)
1840{
1841 flush_workqueue(queue);
1842}
1843
David Chinner585e6d82007-02-10 18:32:29 +11001844/*
1845 * Move as many buffers as specified to the supplied list
1846 * idicating if we skipped any buffers to prevent deadlocks.
1847 */
1848STATIC int
1849xfs_buf_delwri_split(
1850 xfs_buftarg_t *target,
1851 struct list_head *list,
David Chinner5e6a07d2007-02-10 18:34:49 +11001852 unsigned long age)
David Chinner585e6d82007-02-10 18:32:29 +11001853{
1854 xfs_buf_t *bp, *n;
1855 struct list_head *dwq = &target->bt_delwrite_queue;
1856 spinlock_t *dwlk = &target->bt_delwrite_lock;
1857 int skipped = 0;
David Chinner5e6a07d2007-02-10 18:34:49 +11001858 int force;
David Chinner585e6d82007-02-10 18:32:29 +11001859
David Chinner5e6a07d2007-02-10 18:34:49 +11001860 force = test_and_clear_bit(XBT_FORCE_FLUSH, &target->bt_flags);
David Chinner585e6d82007-02-10 18:32:29 +11001861 INIT_LIST_HEAD(list);
1862 spin_lock(dwlk);
1863 list_for_each_entry_safe(bp, n, dwq, b_list) {
David Chinner585e6d82007-02-10 18:32:29 +11001864 ASSERT(bp->b_flags & XBF_DELWRI);
1865
Christoph Hellwig4d16e922010-06-23 18:11:15 +10001866 if (!XFS_BUF_ISPINNED(bp) && !xfs_buf_cond_lock(bp)) {
David Chinner5e6a07d2007-02-10 18:34:49 +11001867 if (!force &&
David Chinner585e6d82007-02-10 18:32:29 +11001868 time_before(jiffies, bp->b_queuetime + age)) {
1869 xfs_buf_unlock(bp);
1870 break;
1871 }
1872
1873 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q|
1874 _XBF_RUN_QUEUES);
1875 bp->b_flags |= XBF_WRITE;
1876 list_move_tail(&bp->b_list, list);
Dave Chinnerbfe27412010-11-08 08:55:05 +00001877 trace_xfs_buf_delwri_split(bp, _RET_IP_);
David Chinner585e6d82007-02-10 18:32:29 +11001878 } else
1879 skipped++;
1880 }
1881 spin_unlock(dwlk);
1882
1883 return skipped;
1884
1885}
1886
Dave Chinner089716a2010-01-26 15:13:25 +11001887/*
1888 * Compare function is more complex than it needs to be because
1889 * the return value is only 32 bits and we are doing comparisons
1890 * on 64 bit values
1891 */
1892static int
1893xfs_buf_cmp(
1894 void *priv,
1895 struct list_head *a,
1896 struct list_head *b)
1897{
1898 struct xfs_buf *ap = container_of(a, struct xfs_buf, b_list);
1899 struct xfs_buf *bp = container_of(b, struct xfs_buf, b_list);
1900 xfs_daddr_t diff;
1901
1902 diff = ap->b_bn - bp->b_bn;
1903 if (diff < 0)
1904 return -1;
1905 if (diff > 0)
1906 return 1;
1907 return 0;
1908}
1909
1910void
1911xfs_buf_delwri_sort(
1912 xfs_buftarg_t *target,
1913 struct list_head *list)
1914{
1915 list_sort(NULL, list, xfs_buf_cmp);
1916}
1917
Linus Torvalds1da177e2005-04-16 15:20:36 -07001918STATIC int
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001919xfsbufd(
David Chinner585e6d82007-02-10 18:32:29 +11001920 void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001921{
Dave Chinner089716a2010-01-26 15:13:25 +11001922 xfs_buftarg_t *target = (xfs_buftarg_t *)data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001923
Linus Torvalds1da177e2005-04-16 15:20:36 -07001924 current->flags |= PF_MEMALLOC;
1925
Rafael J. Wysocki978c7b22007-12-07 14:09:02 +11001926 set_freezable();
1927
Linus Torvalds1da177e2005-04-16 15:20:36 -07001928 do {
Dave Chinnerc9c12972010-01-11 11:49:59 +00001929 long age = xfs_buf_age_centisecs * msecs_to_jiffies(10);
1930 long tout = xfs_buf_timer_centisecs * msecs_to_jiffies(10);
Dave Chinner089716a2010-01-26 15:13:25 +11001931 int count = 0;
1932 struct list_head tmp;
Dave Chinnerc9c12972010-01-11 11:49:59 +00001933
Christoph Lameter3e1d1d22005-06-24 23:13:50 -07001934 if (unlikely(freezing(current))) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001935 set_bit(XBT_FORCE_SLEEP, &target->bt_flags);
Christoph Lameter3e1d1d22005-06-24 23:13:50 -07001936 refrigerator();
Nathan Scottabd0cf72005-05-05 13:30:13 -07001937 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +11001938 clear_bit(XBT_FORCE_SLEEP, &target->bt_flags);
Nathan Scottabd0cf72005-05-05 13:30:13 -07001939 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001940
Dave Chinnerc9c12972010-01-11 11:49:59 +00001941 /* sleep for a long time if there is nothing to do. */
1942 if (list_empty(&target->bt_delwrite_queue))
1943 tout = MAX_SCHEDULE_TIMEOUT;
1944 schedule_timeout_interruptible(tout);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001945
Dave Chinnerc9c12972010-01-11 11:49:59 +00001946 xfs_buf_delwri_split(target, &tmp, age);
Dave Chinner089716a2010-01-26 15:13:25 +11001947 list_sort(NULL, &tmp, xfs_buf_cmp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001948 while (!list_empty(&tmp)) {
Dave Chinner089716a2010-01-26 15:13:25 +11001949 struct xfs_buf *bp;
1950 bp = list_first_entry(&tmp, struct xfs_buf, b_list);
Nathan Scottce8e9222006-01-11 15:39:08 +11001951 list_del_init(&bp->b_list);
Christoph Hellwig939d7232010-07-20 17:51:16 +10001952 xfs_bdstrat_cb(bp);
David Chinner585e6d82007-02-10 18:32:29 +11001953 count++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001954 }
Nathan Scottf07c2252006-09-28 10:52:15 +10001955 if (count)
1956 blk_run_address_space(target->bt_mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001957
Christoph Hellwig4df08c52005-09-05 08:34:18 +10001958 } while (!kthread_should_stop());
Linus Torvalds1da177e2005-04-16 15:20:36 -07001959
Christoph Hellwig4df08c52005-09-05 08:34:18 +10001960 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001961}
1962
1963/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001964 * Go through all incore buffers, and release buffers if they belong to
1965 * the given device. This is used in filesystem error handling to
1966 * preserve the consistency of its metadata.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001967 */
1968int
1969xfs_flush_buftarg(
David Chinner585e6d82007-02-10 18:32:29 +11001970 xfs_buftarg_t *target,
1971 int wait)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001972{
Dave Chinner089716a2010-01-26 15:13:25 +11001973 xfs_buf_t *bp;
David Chinner585e6d82007-02-10 18:32:29 +11001974 int pincount = 0;
Dave Chinner089716a2010-01-26 15:13:25 +11001975 LIST_HEAD(tmp_list);
1976 LIST_HEAD(wait_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001977
Dave Chinnerc626d172009-04-06 18:42:11 +02001978 xfs_buf_runall_queues(xfsconvertd_workqueue);
Nathan Scottce8e9222006-01-11 15:39:08 +11001979 xfs_buf_runall_queues(xfsdatad_workqueue);
1980 xfs_buf_runall_queues(xfslogd_workqueue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001981
David Chinner5e6a07d2007-02-10 18:34:49 +11001982 set_bit(XBT_FORCE_FLUSH, &target->bt_flags);
Dave Chinner089716a2010-01-26 15:13:25 +11001983 pincount = xfs_buf_delwri_split(target, &tmp_list, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001984
1985 /*
Dave Chinner089716a2010-01-26 15:13:25 +11001986 * Dropped the delayed write list lock, now walk the temporary list.
1987 * All I/O is issued async and then if we need to wait for completion
1988 * we do that after issuing all the IO.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001989 */
Dave Chinner089716a2010-01-26 15:13:25 +11001990 list_sort(NULL, &tmp_list, xfs_buf_cmp);
1991 while (!list_empty(&tmp_list)) {
1992 bp = list_first_entry(&tmp_list, struct xfs_buf, b_list);
David Chinner585e6d82007-02-10 18:32:29 +11001993 ASSERT(target == bp->b_target);
Dave Chinner089716a2010-01-26 15:13:25 +11001994 list_del_init(&bp->b_list);
1995 if (wait) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001996 bp->b_flags &= ~XBF_ASYNC;
Dave Chinner089716a2010-01-26 15:13:25 +11001997 list_add(&bp->b_list, &wait_list);
1998 }
Christoph Hellwig939d7232010-07-20 17:51:16 +10001999 xfs_bdstrat_cb(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002000 }
2001
Dave Chinner089716a2010-01-26 15:13:25 +11002002 if (wait) {
2003 /* Expedite and wait for IO to complete. */
Nathan Scottf07c2252006-09-28 10:52:15 +10002004 blk_run_address_space(target->bt_mapping);
Dave Chinner089716a2010-01-26 15:13:25 +11002005 while (!list_empty(&wait_list)) {
2006 bp = list_first_entry(&wait_list, struct xfs_buf, b_list);
Nathan Scottf07c2252006-09-28 10:52:15 +10002007
Dave Chinner089716a2010-01-26 15:13:25 +11002008 list_del_init(&bp->b_list);
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +00002009 xfs_buf_iowait(bp);
Dave Chinner089716a2010-01-26 15:13:25 +11002010 xfs_buf_relse(bp);
2011 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002012 }
2013
Linus Torvalds1da177e2005-04-16 15:20:36 -07002014 return pincount;
2015}
2016
Christoph Hellwig04d8b282005-11-02 10:15:05 +11002017int __init
Nathan Scottce8e9222006-01-11 15:39:08 +11002018xfs_buf_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002019{
Nathan Scott87582802006-03-14 13:18:19 +11002020 xfs_buf_zone = kmem_zone_init_flags(sizeof(xfs_buf_t), "xfs_buf",
2021 KM_ZONE_HWALIGN, NULL);
Nathan Scottce8e9222006-01-11 15:39:08 +11002022 if (!xfs_buf_zone)
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00002023 goto out;
Christoph Hellwig04d8b282005-11-02 10:15:05 +11002024
Dave Chinner51749e42010-09-08 09:00:22 +00002025 xfslogd_workqueue = alloc_workqueue("xfslogd",
Tejun Heo6370a6a2010-10-11 15:12:27 +02002026 WQ_MEM_RECLAIM | WQ_HIGHPRI, 1);
Christoph Hellwig23ea4032005-06-21 15:14:01 +10002027 if (!xfslogd_workqueue)
Christoph Hellwig04d8b282005-11-02 10:15:05 +11002028 goto out_free_buf_zone;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002029
Rafael J. Wysockib4337692007-03-22 00:11:27 -08002030 xfsdatad_workqueue = create_workqueue("xfsdatad");
Christoph Hellwig23ea4032005-06-21 15:14:01 +10002031 if (!xfsdatad_workqueue)
2032 goto out_destroy_xfslogd_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002033
Dave Chinnerc626d172009-04-06 18:42:11 +02002034 xfsconvertd_workqueue = create_workqueue("xfsconvertd");
2035 if (!xfsconvertd_workqueue)
2036 goto out_destroy_xfsdatad_workqueue;
2037
Christoph Hellwig23ea4032005-06-21 15:14:01 +10002038 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002039
Dave Chinnerc626d172009-04-06 18:42:11 +02002040 out_destroy_xfsdatad_workqueue:
2041 destroy_workqueue(xfsdatad_workqueue);
Christoph Hellwig23ea4032005-06-21 15:14:01 +10002042 out_destroy_xfslogd_workqueue:
2043 destroy_workqueue(xfslogd_workqueue);
Christoph Hellwig23ea4032005-06-21 15:14:01 +10002044 out_free_buf_zone:
Nathan Scottce8e9222006-01-11 15:39:08 +11002045 kmem_zone_destroy(xfs_buf_zone);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00002046 out:
Nathan Scott87582802006-03-14 13:18:19 +11002047 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002048}
2049
Linus Torvalds1da177e2005-04-16 15:20:36 -07002050void
Nathan Scottce8e9222006-01-11 15:39:08 +11002051xfs_buf_terminate(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002052{
Dave Chinnerc626d172009-04-06 18:42:11 +02002053 destroy_workqueue(xfsconvertd_workqueue);
Christoph Hellwig04d8b282005-11-02 10:15:05 +11002054 destroy_workqueue(xfsdatad_workqueue);
2055 destroy_workqueue(xfslogd_workqueue);
Nathan Scottce8e9222006-01-11 15:39:08 +11002056 kmem_zone_destroy(xfs_buf_zone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002057}
Tim Shimmine6a0e9c2007-05-08 13:49:59 +10002058
2059#ifdef CONFIG_KDB_MODULES
2060struct list_head *
2061xfs_get_buftarg_list(void)
2062{
2063 return &xfs_buftarg_list;
2064}
2065#endif