blob: 4e8f0df82d025e6bc38615a94d254bf003c24972 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Nathan Scottf07c2252006-09-28 10:52:15 +10002 * Copyright (c) 2000-2006 Silicon Graphics, Inc.
Nathan Scott7b718762005-11-02 14:58:39 +11003 * All Rights Reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
Nathan Scott7b718762005-11-02 14:58:39 +11005 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License as
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 * published by the Free Software Foundation.
8 *
Nathan Scott7b718762005-11-02 14:58:39 +11009 * This program is distributed in the hope that it would be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 *
Nathan Scott7b718762005-11-02 14:58:39 +110014 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
Linus Torvalds1da177e2005-04-16 15:20:36 -070017 */
Vlad Apostolov93c189c2006-11-11 18:03:49 +110018#include "xfs.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/stddef.h>
20#include <linux/errno.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090021#include <linux/gfp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022#include <linux/pagemap.h>
23#include <linux/init.h>
24#include <linux/vmalloc.h>
25#include <linux/bio.h>
26#include <linux/sysctl.h>
27#include <linux/proc_fs.h>
28#include <linux/workqueue.h>
29#include <linux/percpu.h>
30#include <linux/blkdev.h>
31#include <linux/hash.h>
Christoph Hellwig4df08c52005-09-05 08:34:18 +100032#include <linux/kthread.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080033#include <linux/migrate.h>
Andrew Morton3fcfab12006-10-19 23:28:16 -070034#include <linux/backing-dev.h>
Nigel Cunningham7dfb7102006-12-06 20:34:23 -080035#include <linux/freezer.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070036
Christoph Hellwigb7963132009-03-03 14:48:37 -050037#include "xfs_sb.h"
Dave Chinnered3b4d62010-05-21 12:07:08 +100038#include "xfs_log.h"
Christoph Hellwigb7963132009-03-03 14:48:37 -050039#include "xfs_ag.h"
Christoph Hellwigb7963132009-03-03 14:48:37 -050040#include "xfs_mount.h"
Christoph Hellwig0b1b2132009-12-14 23:14:59 +000041#include "xfs_trace.h"
Christoph Hellwigb7963132009-03-03 14:48:37 -050042
David Chinner7989cb82007-02-10 18:34:56 +110043static kmem_zone_t *xfs_buf_zone;
Christoph Hellwig23ea4032005-06-21 15:14:01 +100044
David Chinner7989cb82007-02-10 18:34:56 +110045static struct workqueue_struct *xfslogd_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070046
Nathan Scottce8e9222006-01-11 15:39:08 +110047#ifdef XFS_BUF_LOCK_TRACKING
48# define XB_SET_OWNER(bp) ((bp)->b_last_holder = current->pid)
49# define XB_CLEAR_OWNER(bp) ((bp)->b_last_holder = -1)
50# define XB_GET_OWNER(bp) ((bp)->b_last_holder)
Linus Torvalds1da177e2005-04-16 15:20:36 -070051#else
Nathan Scottce8e9222006-01-11 15:39:08 +110052# define XB_SET_OWNER(bp) do { } while (0)
53# define XB_CLEAR_OWNER(bp) do { } while (0)
54# define XB_GET_OWNER(bp) do { } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -070055#endif
56
Nathan Scottce8e9222006-01-11 15:39:08 +110057#define xb_to_gfp(flags) \
Dave Chinneraa5c1582012-04-23 15:58:56 +100058 ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : GFP_NOFS) | __GFP_NOWARN)
Linus Torvalds1da177e2005-04-16 15:20:36 -070059
Linus Torvalds1da177e2005-04-16 15:20:36 -070060
James Bottomley73c77e22010-01-25 11:42:24 -060061static inline int
62xfs_buf_is_vmapped(
63 struct xfs_buf *bp)
64{
65 /*
66 * Return true if the buffer is vmapped.
67 *
Dave Chinner611c9942012-04-23 15:59:07 +100068 * b_addr is null if the buffer is not mapped, but the code is clever
69 * enough to know it doesn't have to map a single page, so the check has
70 * to be both for b_addr and bp->b_page_count > 1.
James Bottomley73c77e22010-01-25 11:42:24 -060071 */
Dave Chinner611c9942012-04-23 15:59:07 +100072 return bp->b_addr && bp->b_page_count > 1;
James Bottomley73c77e22010-01-25 11:42:24 -060073}
74
75static inline int
76xfs_buf_vmap_len(
77 struct xfs_buf *bp)
78{
79 return (bp->b_page_count * PAGE_SIZE) - bp->b_offset;
80}
81
Linus Torvalds1da177e2005-04-16 15:20:36 -070082/*
Dave Chinner430cbeb2010-12-02 16:30:55 +110083 * xfs_buf_lru_add - add a buffer to the LRU.
84 *
85 * The LRU takes a new reference to the buffer so that it will only be freed
86 * once the shrinker takes the buffer off the LRU.
Linus Torvalds1da177e2005-04-16 15:20:36 -070087 */
Dave Chinner430cbeb2010-12-02 16:30:55 +110088STATIC void
89xfs_buf_lru_add(
90 struct xfs_buf *bp)
91{
92 struct xfs_buftarg *btp = bp->b_target;
93
94 spin_lock(&btp->bt_lru_lock);
95 if (list_empty(&bp->b_lru)) {
96 atomic_inc(&bp->b_hold);
97 list_add_tail(&bp->b_lru, &btp->bt_lru);
98 btp->bt_lru_nr++;
Carlos Maiolino6fb8a902012-08-10 15:01:51 -030099 bp->b_lru_flags &= ~_XBF_LRU_DISPOSE;
Dave Chinner430cbeb2010-12-02 16:30:55 +1100100 }
101 spin_unlock(&btp->bt_lru_lock);
102}
103
104/*
105 * xfs_buf_lru_del - remove a buffer from the LRU
106 *
107 * The unlocked check is safe here because it only occurs when there are not
108 * b_lru_ref counts left on the inode under the pag->pag_buf_lock. it is there
109 * to optimise the shrinker removing the buffer from the LRU and calling
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300110 * xfs_buf_free(). i.e. it removes an unnecessary round trip on the
Dave Chinner430cbeb2010-12-02 16:30:55 +1100111 * bt_lru_lock.
112 */
113STATIC void
114xfs_buf_lru_del(
115 struct xfs_buf *bp)
116{
117 struct xfs_buftarg *btp = bp->b_target;
118
119 if (list_empty(&bp->b_lru))
120 return;
121
122 spin_lock(&btp->bt_lru_lock);
123 if (!list_empty(&bp->b_lru)) {
124 list_del_init(&bp->b_lru);
125 btp->bt_lru_nr--;
126 }
127 spin_unlock(&btp->bt_lru_lock);
128}
129
130/*
131 * When we mark a buffer stale, we remove the buffer from the LRU and clear the
132 * b_lru_ref count so that the buffer is freed immediately when the buffer
133 * reference count falls to zero. If the buffer is already on the LRU, we need
134 * to remove the reference that LRU holds on the buffer.
135 *
136 * This prevents build-up of stale buffers on the LRU.
137 */
138void
139xfs_buf_stale(
140 struct xfs_buf *bp)
141{
Christoph Hellwig43ff2122012-04-23 15:58:39 +1000142 ASSERT(xfs_buf_islocked(bp));
143
Dave Chinner430cbeb2010-12-02 16:30:55 +1100144 bp->b_flags |= XBF_STALE;
Christoph Hellwig43ff2122012-04-23 15:58:39 +1000145
146 /*
147 * Clear the delwri status so that a delwri queue walker will not
148 * flush this buffer to disk now that it is stale. The delwri queue has
149 * a reference to the buffer, so this is safe to do.
150 */
151 bp->b_flags &= ~_XBF_DELWRI_Q;
152
Dave Chinner430cbeb2010-12-02 16:30:55 +1100153 atomic_set(&(bp)->b_lru_ref, 0);
154 if (!list_empty(&bp->b_lru)) {
155 struct xfs_buftarg *btp = bp->b_target;
156
157 spin_lock(&btp->bt_lru_lock);
Carlos Maiolino6fb8a902012-08-10 15:01:51 -0300158 if (!list_empty(&bp->b_lru) &&
159 !(bp->b_lru_flags & _XBF_LRU_DISPOSE)) {
Dave Chinner430cbeb2010-12-02 16:30:55 +1100160 list_del_init(&bp->b_lru);
161 btp->bt_lru_nr--;
162 atomic_dec(&bp->b_hold);
163 }
164 spin_unlock(&btp->bt_lru_lock);
165 }
166 ASSERT(atomic_read(&bp->b_hold) >= 1);
167}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168
Dave Chinner3e85c862012-06-22 18:50:09 +1000169static int
170xfs_buf_get_maps(
171 struct xfs_buf *bp,
172 int map_count)
173{
174 ASSERT(bp->b_maps == NULL);
175 bp->b_map_count = map_count;
176
177 if (map_count == 1) {
Mark Tinguelyf4b42422012-12-04 17:18:02 -0600178 bp->b_maps = &bp->__b_map;
Dave Chinner3e85c862012-06-22 18:50:09 +1000179 return 0;
180 }
181
182 bp->b_maps = kmem_zalloc(map_count * sizeof(struct xfs_buf_map),
183 KM_NOFS);
184 if (!bp->b_maps)
185 return ENOMEM;
186 return 0;
187}
188
189/*
190 * Frees b_pages if it was allocated.
191 */
192static void
193xfs_buf_free_maps(
194 struct xfs_buf *bp)
195{
Mark Tinguelyf4b42422012-12-04 17:18:02 -0600196 if (bp->b_maps != &bp->__b_map) {
Dave Chinner3e85c862012-06-22 18:50:09 +1000197 kmem_free(bp->b_maps);
198 bp->b_maps = NULL;
199 }
200}
201
Christoph Hellwig4347b9d2011-10-10 16:52:48 +0000202struct xfs_buf *
Dave Chinner3e85c862012-06-22 18:50:09 +1000203_xfs_buf_alloc(
Christoph Hellwig4347b9d2011-10-10 16:52:48 +0000204 struct xfs_buftarg *target,
Dave Chinner3e85c862012-06-22 18:50:09 +1000205 struct xfs_buf_map *map,
206 int nmaps,
Nathan Scottce8e9222006-01-11 15:39:08 +1100207 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208{
Christoph Hellwig4347b9d2011-10-10 16:52:48 +0000209 struct xfs_buf *bp;
Dave Chinner3e85c862012-06-22 18:50:09 +1000210 int error;
211 int i;
Christoph Hellwig4347b9d2011-10-10 16:52:48 +0000212
Dave Chinneraa5c1582012-04-23 15:58:56 +1000213 bp = kmem_zone_zalloc(xfs_buf_zone, KM_NOFS);
Christoph Hellwig4347b9d2011-10-10 16:52:48 +0000214 if (unlikely(!bp))
215 return NULL;
216
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217 /*
Dave Chinner12bcb3f2012-04-23 15:59:05 +1000218 * We don't want certain flags to appear in b_flags unless they are
219 * specifically set by later operations on the buffer.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220 */
Dave Chinner611c9942012-04-23 15:59:07 +1000221 flags &= ~(XBF_UNMAPPED | XBF_TRYLOCK | XBF_ASYNC | XBF_READ_AHEAD);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222
Nathan Scottce8e9222006-01-11 15:39:08 +1100223 atomic_set(&bp->b_hold, 1);
Dave Chinner430cbeb2010-12-02 16:30:55 +1100224 atomic_set(&bp->b_lru_ref, 1);
David Chinnerb4dd3302008-08-13 16:36:11 +1000225 init_completion(&bp->b_iowait);
Dave Chinner430cbeb2010-12-02 16:30:55 +1100226 INIT_LIST_HEAD(&bp->b_lru);
Nathan Scottce8e9222006-01-11 15:39:08 +1100227 INIT_LIST_HEAD(&bp->b_list);
Dave Chinner74f75a02010-09-24 19:59:04 +1000228 RB_CLEAR_NODE(&bp->b_rbnode);
Thomas Gleixnera731cd12010-09-07 14:33:15 +0000229 sema_init(&bp->b_sema, 0); /* held, no waiters */
Nathan Scottce8e9222006-01-11 15:39:08 +1100230 XB_SET_OWNER(bp);
231 bp->b_target = target;
Dave Chinner3e85c862012-06-22 18:50:09 +1000232 bp->b_flags = flags;
Dave Chinnerde1cbee2012-04-23 15:58:50 +1000233
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234 /*
Dave Chinneraa0e8832012-04-23 15:58:52 +1000235 * Set length and io_length to the same value initially.
236 * I/O routines should use io_length, which will be the same in
Linus Torvalds1da177e2005-04-16 15:20:36 -0700237 * most cases but may be reset (e.g. XFS recovery).
238 */
Dave Chinner3e85c862012-06-22 18:50:09 +1000239 error = xfs_buf_get_maps(bp, nmaps);
240 if (error) {
241 kmem_zone_free(xfs_buf_zone, bp);
242 return NULL;
243 }
244
245 bp->b_bn = map[0].bm_bn;
246 bp->b_length = 0;
247 for (i = 0; i < nmaps; i++) {
248 bp->b_maps[i].bm_bn = map[i].bm_bn;
249 bp->b_maps[i].bm_len = map[i].bm_len;
250 bp->b_length += map[i].bm_len;
251 }
252 bp->b_io_length = bp->b_length;
253
Nathan Scottce8e9222006-01-11 15:39:08 +1100254 atomic_set(&bp->b_pin_count, 0);
255 init_waitqueue_head(&bp->b_waiters);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256
Nathan Scottce8e9222006-01-11 15:39:08 +1100257 XFS_STATS_INC(xb_create);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000258 trace_xfs_buf_init(bp, _RET_IP_);
Christoph Hellwig4347b9d2011-10-10 16:52:48 +0000259
260 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700261}
262
263/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100264 * Allocate a page array capable of holding a specified number
265 * of pages, and point the page buf at it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266 */
267STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100268_xfs_buf_get_pages(
269 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270 int page_count,
Nathan Scottce8e9222006-01-11 15:39:08 +1100271 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272{
273 /* Make sure that we have a page list */
Nathan Scottce8e9222006-01-11 15:39:08 +1100274 if (bp->b_pages == NULL) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100275 bp->b_page_count = page_count;
276 if (page_count <= XB_PAGES) {
277 bp->b_pages = bp->b_page_array;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100279 bp->b_pages = kmem_alloc(sizeof(struct page *) *
Dave Chinneraa5c1582012-04-23 15:58:56 +1000280 page_count, KM_NOFS);
Nathan Scottce8e9222006-01-11 15:39:08 +1100281 if (bp->b_pages == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282 return -ENOMEM;
283 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100284 memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700285 }
286 return 0;
287}
288
289/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100290 * Frees b_pages if it was allocated.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291 */
292STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +1100293_xfs_buf_free_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700294 xfs_buf_t *bp)
295{
Nathan Scottce8e9222006-01-11 15:39:08 +1100296 if (bp->b_pages != bp->b_page_array) {
Denys Vlasenkof0e2d932008-05-19 16:31:57 +1000297 kmem_free(bp->b_pages);
Dave Chinner3fc98b12009-12-14 23:11:57 +0000298 bp->b_pages = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299 }
300}
301
302/*
303 * Releases the specified buffer.
304 *
305 * The modification state of any associated pages is left unchanged.
Nathan Scottce8e9222006-01-11 15:39:08 +1100306 * The buffer most not be on any hash - use xfs_buf_rele instead for
Linus Torvalds1da177e2005-04-16 15:20:36 -0700307 * hashed and refcounted buffers
308 */
309void
Nathan Scottce8e9222006-01-11 15:39:08 +1100310xfs_buf_free(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311 xfs_buf_t *bp)
312{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000313 trace_xfs_buf_free(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314
Dave Chinner430cbeb2010-12-02 16:30:55 +1100315 ASSERT(list_empty(&bp->b_lru));
316
Dave Chinner0e6e8472011-03-26 09:16:45 +1100317 if (bp->b_flags & _XBF_PAGES) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318 uint i;
319
James Bottomley73c77e22010-01-25 11:42:24 -0600320 if (xfs_buf_is_vmapped(bp))
Alex Elder8a262e52010-03-16 18:55:56 +0000321 vm_unmap_ram(bp->b_addr - bp->b_offset,
322 bp->b_page_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323
Nathan Scott948ecdb2006-09-28 11:03:13 +1000324 for (i = 0; i < bp->b_page_count; i++) {
325 struct page *page = bp->b_pages[i];
326
Dave Chinner0e6e8472011-03-26 09:16:45 +1100327 __free_page(page);
Nathan Scott948ecdb2006-09-28 11:03:13 +1000328 }
Dave Chinner0e6e8472011-03-26 09:16:45 +1100329 } else if (bp->b_flags & _XBF_KMEM)
330 kmem_free(bp->b_addr);
Dave Chinner3fc98b12009-12-14 23:11:57 +0000331 _xfs_buf_free_pages(bp);
Dave Chinner3e85c862012-06-22 18:50:09 +1000332 xfs_buf_free_maps(bp);
Christoph Hellwig4347b9d2011-10-10 16:52:48 +0000333 kmem_zone_free(xfs_buf_zone, bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700334}
335
336/*
Dave Chinner0e6e8472011-03-26 09:16:45 +1100337 * Allocates all the pages for buffer in question and builds it's page list.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700338 */
339STATIC int
Dave Chinner0e6e8472011-03-26 09:16:45 +1100340xfs_buf_allocate_memory(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700341 xfs_buf_t *bp,
342 uint flags)
343{
Dave Chinneraa0e8832012-04-23 15:58:52 +1000344 size_t size;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700345 size_t nbytes, offset;
Nathan Scottce8e9222006-01-11 15:39:08 +1100346 gfp_t gfp_mask = xb_to_gfp(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347 unsigned short page_count, i;
Dave Chinner795cac72012-04-23 15:58:53 +1000348 xfs_off_t start, end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700349 int error;
350
Dave Chinner0e6e8472011-03-26 09:16:45 +1100351 /*
352 * for buffers that are contained within a single page, just allocate
353 * the memory from the heap - there's no need for the complexity of
354 * page arrays to keep allocation down to order 0.
355 */
Dave Chinner795cac72012-04-23 15:58:53 +1000356 size = BBTOB(bp->b_length);
357 if (size < PAGE_SIZE) {
Dave Chinneraa5c1582012-04-23 15:58:56 +1000358 bp->b_addr = kmem_alloc(size, KM_NOFS);
Dave Chinner0e6e8472011-03-26 09:16:45 +1100359 if (!bp->b_addr) {
360 /* low memory - use alloc_page loop instead */
361 goto use_alloc_page;
362 }
363
Dave Chinner795cac72012-04-23 15:58:53 +1000364 if (((unsigned long)(bp->b_addr + size - 1) & PAGE_MASK) !=
Dave Chinner0e6e8472011-03-26 09:16:45 +1100365 ((unsigned long)bp->b_addr & PAGE_MASK)) {
366 /* b_addr spans two pages - use alloc_page instead */
367 kmem_free(bp->b_addr);
368 bp->b_addr = NULL;
369 goto use_alloc_page;
370 }
371 bp->b_offset = offset_in_page(bp->b_addr);
372 bp->b_pages = bp->b_page_array;
373 bp->b_pages[0] = virt_to_page(bp->b_addr);
374 bp->b_page_count = 1;
Dave Chinner611c9942012-04-23 15:59:07 +1000375 bp->b_flags |= _XBF_KMEM;
Dave Chinner0e6e8472011-03-26 09:16:45 +1100376 return 0;
377 }
378
379use_alloc_page:
Mark Tinguelyf4b42422012-12-04 17:18:02 -0600380 start = BBTOB(bp->b_maps[0].bm_bn) >> PAGE_SHIFT;
381 end = (BBTOB(bp->b_maps[0].bm_bn + bp->b_length) + PAGE_SIZE - 1)
Dave Chinnercbb7baab2012-06-22 18:50:08 +1000382 >> PAGE_SHIFT;
Dave Chinner795cac72012-04-23 15:58:53 +1000383 page_count = end - start;
Nathan Scottce8e9222006-01-11 15:39:08 +1100384 error = _xfs_buf_get_pages(bp, page_count, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385 if (unlikely(error))
386 return error;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700387
Nathan Scottce8e9222006-01-11 15:39:08 +1100388 offset = bp->b_offset;
Dave Chinner0e6e8472011-03-26 09:16:45 +1100389 bp->b_flags |= _XBF_PAGES;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390
Nathan Scottce8e9222006-01-11 15:39:08 +1100391 for (i = 0; i < bp->b_page_count; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700392 struct page *page;
393 uint retries = 0;
Dave Chinner0e6e8472011-03-26 09:16:45 +1100394retry:
395 page = alloc_page(gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396 if (unlikely(page == NULL)) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100397 if (flags & XBF_READ_AHEAD) {
398 bp->b_page_count = i;
Dave Chinner0e6e8472011-03-26 09:16:45 +1100399 error = ENOMEM;
400 goto out_free_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401 }
402
403 /*
404 * This could deadlock.
405 *
406 * But until all the XFS lowlevel code is revamped to
407 * handle buffer allocation failures we can't do much.
408 */
409 if (!(++retries % 100))
Dave Chinner4f107002011-03-07 10:00:35 +1100410 xfs_err(NULL,
411 "possible memory allocation deadlock in %s (mode:0x%x)",
Harvey Harrison34a622b2008-04-10 12:19:21 +1000412 __func__, gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413
Nathan Scottce8e9222006-01-11 15:39:08 +1100414 XFS_STATS_INC(xb_page_retries);
Jens Axboe8aa7e842009-07-09 14:52:32 +0200415 congestion_wait(BLK_RW_ASYNC, HZ/50);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416 goto retry;
417 }
418
Nathan Scottce8e9222006-01-11 15:39:08 +1100419 XFS_STATS_INC(xb_page_found);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420
Dave Chinner0e6e8472011-03-26 09:16:45 +1100421 nbytes = min_t(size_t, size, PAGE_SIZE - offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422 size -= nbytes;
Nathan Scottce8e9222006-01-11 15:39:08 +1100423 bp->b_pages[i] = page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424 offset = 0;
425 }
Dave Chinner0e6e8472011-03-26 09:16:45 +1100426 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427
Dave Chinner0e6e8472011-03-26 09:16:45 +1100428out_free_pages:
429 for (i = 0; i < bp->b_page_count; i++)
430 __free_page(bp->b_pages[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700431 return error;
432}
433
434/*
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300435 * Map buffer into kernel address-space if necessary.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436 */
437STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100438_xfs_buf_map_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700439 xfs_buf_t *bp,
440 uint flags)
441{
Dave Chinner0e6e8472011-03-26 09:16:45 +1100442 ASSERT(bp->b_flags & _XBF_PAGES);
Nathan Scottce8e9222006-01-11 15:39:08 +1100443 if (bp->b_page_count == 1) {
Dave Chinner0e6e8472011-03-26 09:16:45 +1100444 /* A single page buffer is always mappable */
Nathan Scottce8e9222006-01-11 15:39:08 +1100445 bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
Dave Chinner611c9942012-04-23 15:59:07 +1000446 } else if (flags & XBF_UNMAPPED) {
447 bp->b_addr = NULL;
448 } else {
Dave Chinnera19fb382011-03-26 09:13:42 +1100449 int retried = 0;
450
451 do {
452 bp->b_addr = vm_map_ram(bp->b_pages, bp->b_page_count,
453 -1, PAGE_KERNEL);
454 if (bp->b_addr)
455 break;
456 vm_unmap_aliases();
457 } while (retried++ <= 1);
458
459 if (!bp->b_addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700460 return -ENOMEM;
Nathan Scottce8e9222006-01-11 15:39:08 +1100461 bp->b_addr += bp->b_offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700462 }
463
464 return 0;
465}
466
467/*
468 * Finding and Reading Buffers
469 */
470
471/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100472 * Look up, and creates if absent, a lockable buffer for
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473 * a given range of an inode. The buffer is returned
Chandra Seetharamaneabbaf12011-09-08 20:18:50 +0000474 * locked. No I/O is implied by this call.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700475 */
476xfs_buf_t *
Nathan Scottce8e9222006-01-11 15:39:08 +1100477_xfs_buf_find(
Dave Chinnere70b73f2012-04-23 15:58:49 +1000478 struct xfs_buftarg *btp,
Dave Chinner3e85c862012-06-22 18:50:09 +1000479 struct xfs_buf_map *map,
480 int nmaps,
Nathan Scottce8e9222006-01-11 15:39:08 +1100481 xfs_buf_flags_t flags,
482 xfs_buf_t *new_bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483{
Dave Chinnere70b73f2012-04-23 15:58:49 +1000484 size_t numbytes;
Dave Chinner74f75a02010-09-24 19:59:04 +1000485 struct xfs_perag *pag;
486 struct rb_node **rbp;
487 struct rb_node *parent;
488 xfs_buf_t *bp;
Dave Chinner3e85c862012-06-22 18:50:09 +1000489 xfs_daddr_t blkno = map[0].bm_bn;
Dave Chinner10616b82013-01-21 23:53:52 +1100490 xfs_daddr_t eofs;
Dave Chinner3e85c862012-06-22 18:50:09 +1000491 int numblks = 0;
492 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700493
Dave Chinner3e85c862012-06-22 18:50:09 +1000494 for (i = 0; i < nmaps; i++)
495 numblks += map[i].bm_len;
Dave Chinnere70b73f2012-04-23 15:58:49 +1000496 numbytes = BBTOB(numblks);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497
498 /* Check for IOs smaller than the sector size / not sector aligned */
Dave Chinnere70b73f2012-04-23 15:58:49 +1000499 ASSERT(!(numbytes < (1 << btp->bt_sshift)));
Dave Chinnerde1cbee2012-04-23 15:58:50 +1000500 ASSERT(!(BBTOB(blkno) & (xfs_off_t)btp->bt_smask));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700501
Dave Chinner10616b82013-01-21 23:53:52 +1100502 /*
503 * Corrupted block numbers can get through to here, unfortunately, so we
504 * have to check that the buffer falls within the filesystem bounds.
505 */
506 eofs = XFS_FSB_TO_BB(btp->bt_mount, btp->bt_mount->m_sb.sb_dblocks);
507 if (blkno >= eofs) {
508 /*
509 * XXX (dgc): we should really be returning EFSCORRUPTED here,
510 * but none of the higher level infrastructure supports
511 * returning a specific error on buffer lookup failures.
512 */
513 xfs_alert(btp->bt_mount,
514 "%s: Block out of range: block 0x%llx, EOFS 0x%llx ",
515 __func__, blkno, eofs);
516 return NULL;
517 }
518
Dave Chinner74f75a02010-09-24 19:59:04 +1000519 /* get tree root */
520 pag = xfs_perag_get(btp->bt_mount,
Dave Chinnere70b73f2012-04-23 15:58:49 +1000521 xfs_daddr_to_agno(btp->bt_mount, blkno));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700522
Dave Chinner74f75a02010-09-24 19:59:04 +1000523 /* walk tree */
524 spin_lock(&pag->pag_buf_lock);
525 rbp = &pag->pag_buf_tree.rb_node;
526 parent = NULL;
527 bp = NULL;
528 while (*rbp) {
529 parent = *rbp;
530 bp = rb_entry(parent, struct xfs_buf, b_rbnode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700531
Dave Chinnerde1cbee2012-04-23 15:58:50 +1000532 if (blkno < bp->b_bn)
Dave Chinner74f75a02010-09-24 19:59:04 +1000533 rbp = &(*rbp)->rb_left;
Dave Chinnerde1cbee2012-04-23 15:58:50 +1000534 else if (blkno > bp->b_bn)
Dave Chinner74f75a02010-09-24 19:59:04 +1000535 rbp = &(*rbp)->rb_right;
536 else {
537 /*
Dave Chinnerde1cbee2012-04-23 15:58:50 +1000538 * found a block number match. If the range doesn't
Dave Chinner74f75a02010-09-24 19:59:04 +1000539 * match, the only way this is allowed is if the buffer
540 * in the cache is stale and the transaction that made
541 * it stale has not yet committed. i.e. we are
542 * reallocating a busy extent. Skip this buffer and
543 * continue searching to the right for an exact match.
544 */
Dave Chinner4e94b712012-04-23 15:58:51 +1000545 if (bp->b_length != numblks) {
Dave Chinner74f75a02010-09-24 19:59:04 +1000546 ASSERT(bp->b_flags & XBF_STALE);
547 rbp = &(*rbp)->rb_right;
548 continue;
549 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100550 atomic_inc(&bp->b_hold);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700551 goto found;
552 }
553 }
554
555 /* No match found */
Nathan Scottce8e9222006-01-11 15:39:08 +1100556 if (new_bp) {
Dave Chinner74f75a02010-09-24 19:59:04 +1000557 rb_link_node(&new_bp->b_rbnode, parent, rbp);
558 rb_insert_color(&new_bp->b_rbnode, &pag->pag_buf_tree);
559 /* the buffer keeps the perag reference until it is freed */
560 new_bp->b_pag = pag;
561 spin_unlock(&pag->pag_buf_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700562 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100563 XFS_STATS_INC(xb_miss_locked);
Dave Chinner74f75a02010-09-24 19:59:04 +1000564 spin_unlock(&pag->pag_buf_lock);
565 xfs_perag_put(pag);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700566 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100567 return new_bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700568
569found:
Dave Chinner74f75a02010-09-24 19:59:04 +1000570 spin_unlock(&pag->pag_buf_lock);
571 xfs_perag_put(pag);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572
Christoph Hellwig0c842ad2011-07-08 14:36:19 +0200573 if (!xfs_buf_trylock(bp)) {
574 if (flags & XBF_TRYLOCK) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100575 xfs_buf_rele(bp);
576 XFS_STATS_INC(xb_busy_locked);
577 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578 }
Christoph Hellwig0c842ad2011-07-08 14:36:19 +0200579 xfs_buf_lock(bp);
580 XFS_STATS_INC(xb_get_locked_waited);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700581 }
582
Dave Chinner0e6e8472011-03-26 09:16:45 +1100583 /*
584 * if the buffer is stale, clear all the external state associated with
585 * it. We need to keep flags such as how we allocated the buffer memory
586 * intact here.
587 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100588 if (bp->b_flags & XBF_STALE) {
589 ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
Dave Chinnercfb02852012-11-12 22:54:19 +1100590 ASSERT(bp->b_iodone == NULL);
Dave Chinner611c9942012-04-23 15:59:07 +1000591 bp->b_flags &= _XBF_KMEM | _XBF_PAGES;
Dave Chinner1813dd62012-11-14 17:54:40 +1100592 bp->b_ops = NULL;
David Chinner2f926582005-09-05 08:33:35 +1000593 }
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000594
595 trace_xfs_buf_find(bp, flags, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +1100596 XFS_STATS_INC(xb_get_locked);
597 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598}
599
600/*
Dave Chinner38158322011-09-30 04:45:02 +0000601 * Assembles a buffer covering the specified range. The code is optimised for
602 * cache hits, as metadata intensive workloads will see 3 orders of magnitude
603 * more hits than misses.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700604 */
Dave Chinner38158322011-09-30 04:45:02 +0000605struct xfs_buf *
Dave Chinner6dde2702012-06-22 18:50:10 +1000606xfs_buf_get_map(
607 struct xfs_buftarg *target,
608 struct xfs_buf_map *map,
609 int nmaps,
Nathan Scottce8e9222006-01-11 15:39:08 +1100610 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611{
Dave Chinner38158322011-09-30 04:45:02 +0000612 struct xfs_buf *bp;
613 struct xfs_buf *new_bp;
Dave Chinner0e6e8472011-03-26 09:16:45 +1100614 int error = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615
Dave Chinner6dde2702012-06-22 18:50:10 +1000616 bp = _xfs_buf_find(target, map, nmaps, flags, NULL);
Dave Chinner38158322011-09-30 04:45:02 +0000617 if (likely(bp))
618 goto found;
619
Dave Chinner6dde2702012-06-22 18:50:10 +1000620 new_bp = _xfs_buf_alloc(target, map, nmaps, flags);
Nathan Scottce8e9222006-01-11 15:39:08 +1100621 if (unlikely(!new_bp))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622 return NULL;
623
Dave Chinnerfe2429b2012-04-23 15:58:45 +1000624 error = xfs_buf_allocate_memory(new_bp, flags);
625 if (error) {
Dave Chinner3e85c862012-06-22 18:50:09 +1000626 xfs_buf_free(new_bp);
Dave Chinner38158322011-09-30 04:45:02 +0000627 return NULL;
628 }
629
Dave Chinner6dde2702012-06-22 18:50:10 +1000630 bp = _xfs_buf_find(target, map, nmaps, flags, new_bp);
Dave Chinnerfe2429b2012-04-23 15:58:45 +1000631 if (!bp) {
632 xfs_buf_free(new_bp);
633 return NULL;
634 }
635
636 if (bp != new_bp)
637 xfs_buf_free(new_bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638
Dave Chinner38158322011-09-30 04:45:02 +0000639found:
Dave Chinner611c9942012-04-23 15:59:07 +1000640 if (!bp->b_addr) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100641 error = _xfs_buf_map_pages(bp, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700642 if (unlikely(error)) {
Dave Chinner4f107002011-03-07 10:00:35 +1100643 xfs_warn(target->bt_mount,
644 "%s: failed to map pages\n", __func__);
Dave Chinnera8acad72012-04-23 15:58:54 +1000645 xfs_buf_relse(bp);
646 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647 }
648 }
649
Nathan Scottce8e9222006-01-11 15:39:08 +1100650 XFS_STATS_INC(xb_get);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000651 trace_xfs_buf_get(bp, flags, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +1100652 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700653}
654
Christoph Hellwig5d765b92008-12-03 12:20:26 +0100655STATIC int
656_xfs_buf_read(
657 xfs_buf_t *bp,
658 xfs_buf_flags_t flags)
659{
Christoph Hellwig43ff2122012-04-23 15:58:39 +1000660 ASSERT(!(flags & XBF_WRITE));
Mark Tinguelyf4b42422012-12-04 17:18:02 -0600661 ASSERT(bp->b_maps[0].bm_bn != XFS_BUF_DADDR_NULL);
Christoph Hellwig5d765b92008-12-03 12:20:26 +0100662
Christoph Hellwig43ff2122012-04-23 15:58:39 +1000663 bp->b_flags &= ~(XBF_WRITE | XBF_ASYNC | XBF_READ_AHEAD);
Christoph Hellwig1d5ae5d2011-07-08 14:36:32 +0200664 bp->b_flags |= flags & (XBF_READ | XBF_ASYNC | XBF_READ_AHEAD);
Christoph Hellwig5d765b92008-12-03 12:20:26 +0100665
Dave Chinner0e95f192012-04-23 15:58:46 +1000666 xfs_buf_iorequest(bp);
667 if (flags & XBF_ASYNC)
668 return 0;
Dave Chinnerec53d1d2010-07-20 17:52:59 +1000669 return xfs_buf_iowait(bp);
Christoph Hellwig5d765b92008-12-03 12:20:26 +0100670}
671
Linus Torvalds1da177e2005-04-16 15:20:36 -0700672xfs_buf_t *
Dave Chinner6dde2702012-06-22 18:50:10 +1000673xfs_buf_read_map(
674 struct xfs_buftarg *target,
675 struct xfs_buf_map *map,
676 int nmaps,
Dave Chinnerc3f8fc72012-11-12 22:54:01 +1100677 xfs_buf_flags_t flags,
Dave Chinner1813dd62012-11-14 17:54:40 +1100678 const struct xfs_buf_ops *ops)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700679{
Dave Chinner6dde2702012-06-22 18:50:10 +1000680 struct xfs_buf *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700681
Nathan Scottce8e9222006-01-11 15:39:08 +1100682 flags |= XBF_READ;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683
Dave Chinner6dde2702012-06-22 18:50:10 +1000684 bp = xfs_buf_get_map(target, map, nmaps, flags);
Nathan Scottce8e9222006-01-11 15:39:08 +1100685 if (bp) {
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000686 trace_xfs_buf_read(bp, flags, _RET_IP_);
687
Nathan Scottce8e9222006-01-11 15:39:08 +1100688 if (!XFS_BUF_ISDONE(bp)) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100689 XFS_STATS_INC(xb_get_read);
Dave Chinner1813dd62012-11-14 17:54:40 +1100690 bp->b_ops = ops;
Christoph Hellwig5d765b92008-12-03 12:20:26 +0100691 _xfs_buf_read(bp, flags);
Nathan Scottce8e9222006-01-11 15:39:08 +1100692 } else if (flags & XBF_ASYNC) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700693 /*
694 * Read ahead call which is already satisfied,
695 * drop the buffer
696 */
Dave Chinnera8acad72012-04-23 15:58:54 +1000697 xfs_buf_relse(bp);
698 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700699 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700700 /* We do not want read in the flags */
Nathan Scottce8e9222006-01-11 15:39:08 +1100701 bp->b_flags &= ~XBF_READ;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700702 }
703 }
704
Nathan Scottce8e9222006-01-11 15:39:08 +1100705 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706}
707
708/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100709 * If we are not low on memory then do the readahead in a deadlock
710 * safe manner.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700711 */
712void
Dave Chinner6dde2702012-06-22 18:50:10 +1000713xfs_buf_readahead_map(
714 struct xfs_buftarg *target,
715 struct xfs_buf_map *map,
Dave Chinnerc3f8fc72012-11-12 22:54:01 +1100716 int nmaps,
Dave Chinner1813dd62012-11-14 17:54:40 +1100717 const struct xfs_buf_ops *ops)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700718{
Dave Chinner0e6e8472011-03-26 09:16:45 +1100719 if (bdi_read_congested(target->bt_bdi))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700720 return;
721
Dave Chinner6dde2702012-06-22 18:50:10 +1000722 xfs_buf_read_map(target, map, nmaps,
Dave Chinner1813dd62012-11-14 17:54:40 +1100723 XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD, ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700724}
725
Dave Chinner5adc94c2010-09-24 21:58:31 +1000726/*
727 * Read an uncached buffer from disk. Allocates and returns a locked
728 * buffer containing the disk contents or nothing.
729 */
730struct xfs_buf *
731xfs_buf_read_uncached(
Dave Chinner5adc94c2010-09-24 21:58:31 +1000732 struct xfs_buftarg *target,
733 xfs_daddr_t daddr,
Dave Chinnere70b73f2012-04-23 15:58:49 +1000734 size_t numblks,
Dave Chinnerc3f8fc72012-11-12 22:54:01 +1100735 int flags,
Dave Chinner1813dd62012-11-14 17:54:40 +1100736 const struct xfs_buf_ops *ops)
Dave Chinner5adc94c2010-09-24 21:58:31 +1000737{
Dave Chinnereab4e632012-11-12 22:54:02 +1100738 struct xfs_buf *bp;
Dave Chinner5adc94c2010-09-24 21:58:31 +1000739
Dave Chinnere70b73f2012-04-23 15:58:49 +1000740 bp = xfs_buf_get_uncached(target, numblks, flags);
Dave Chinner5adc94c2010-09-24 21:58:31 +1000741 if (!bp)
742 return NULL;
743
744 /* set up the buffer for a read IO */
Dave Chinner3e85c862012-06-22 18:50:09 +1000745 ASSERT(bp->b_map_count == 1);
746 bp->b_bn = daddr;
747 bp->b_maps[0].bm_bn = daddr;
Dave Chinnercbb7baab2012-06-22 18:50:08 +1000748 bp->b_flags |= XBF_READ;
Dave Chinner1813dd62012-11-14 17:54:40 +1100749 bp->b_ops = ops;
Dave Chinner5adc94c2010-09-24 21:58:31 +1000750
Dave Chinnere70b73f2012-04-23 15:58:49 +1000751 xfsbdstrat(target->bt_mount, bp);
Dave Chinnereab4e632012-11-12 22:54:02 +1100752 xfs_buf_iowait(bp);
Dave Chinner5adc94c2010-09-24 21:58:31 +1000753 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700754}
755
Dave Chinner44396472011-04-21 09:34:27 +0000756/*
757 * Return a buffer allocated as an empty buffer and associated to external
758 * memory via xfs_buf_associate_memory() back to it's empty state.
759 */
760void
761xfs_buf_set_empty(
762 struct xfs_buf *bp,
Dave Chinnere70b73f2012-04-23 15:58:49 +1000763 size_t numblks)
Dave Chinner44396472011-04-21 09:34:27 +0000764{
765 if (bp->b_pages)
766 _xfs_buf_free_pages(bp);
767
768 bp->b_pages = NULL;
769 bp->b_page_count = 0;
770 bp->b_addr = NULL;
Dave Chinner4e94b712012-04-23 15:58:51 +1000771 bp->b_length = numblks;
Dave Chinneraa0e8832012-04-23 15:58:52 +1000772 bp->b_io_length = numblks;
Dave Chinner3e85c862012-06-22 18:50:09 +1000773
774 ASSERT(bp->b_map_count == 1);
Dave Chinner44396472011-04-21 09:34:27 +0000775 bp->b_bn = XFS_BUF_DADDR_NULL;
Dave Chinner3e85c862012-06-22 18:50:09 +1000776 bp->b_maps[0].bm_bn = XFS_BUF_DADDR_NULL;
777 bp->b_maps[0].bm_len = bp->b_length;
Dave Chinner44396472011-04-21 09:34:27 +0000778}
779
Linus Torvalds1da177e2005-04-16 15:20:36 -0700780static inline struct page *
781mem_to_page(
782 void *addr)
783{
Christoph Lameter9e2779f2008-02-04 22:28:34 -0800784 if ((!is_vmalloc_addr(addr))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700785 return virt_to_page(addr);
786 } else {
787 return vmalloc_to_page(addr);
788 }
789}
790
791int
Nathan Scottce8e9222006-01-11 15:39:08 +1100792xfs_buf_associate_memory(
793 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700794 void *mem,
795 size_t len)
796{
797 int rval;
798 int i = 0;
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100799 unsigned long pageaddr;
800 unsigned long offset;
801 size_t buflen;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802 int page_count;
803
Dave Chinner0e6e8472011-03-26 09:16:45 +1100804 pageaddr = (unsigned long)mem & PAGE_MASK;
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100805 offset = (unsigned long)mem - pageaddr;
Dave Chinner0e6e8472011-03-26 09:16:45 +1100806 buflen = PAGE_ALIGN(len + offset);
807 page_count = buflen >> PAGE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700808
809 /* Free any previous set of page pointers */
Nathan Scottce8e9222006-01-11 15:39:08 +1100810 if (bp->b_pages)
811 _xfs_buf_free_pages(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812
Nathan Scottce8e9222006-01-11 15:39:08 +1100813 bp->b_pages = NULL;
814 bp->b_addr = mem;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815
Dave Chinneraa5c1582012-04-23 15:58:56 +1000816 rval = _xfs_buf_get_pages(bp, page_count, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700817 if (rval)
818 return rval;
819
Nathan Scottce8e9222006-01-11 15:39:08 +1100820 bp->b_offset = offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100822 for (i = 0; i < bp->b_page_count; i++) {
823 bp->b_pages[i] = mem_to_page((void *)pageaddr);
Dave Chinner0e6e8472011-03-26 09:16:45 +1100824 pageaddr += PAGE_SIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700825 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700826
Dave Chinneraa0e8832012-04-23 15:58:52 +1000827 bp->b_io_length = BTOBB(len);
Dave Chinner4e94b712012-04-23 15:58:51 +1000828 bp->b_length = BTOBB(buflen);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829
830 return 0;
831}
832
833xfs_buf_t *
Dave Chinner686865f2010-09-24 20:07:47 +1000834xfs_buf_get_uncached(
835 struct xfs_buftarg *target,
Dave Chinnere70b73f2012-04-23 15:58:49 +1000836 size_t numblks,
Dave Chinner686865f2010-09-24 20:07:47 +1000837 int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700838{
Dave Chinnere70b73f2012-04-23 15:58:49 +1000839 unsigned long page_count;
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000840 int error, i;
Dave Chinner3e85c862012-06-22 18:50:09 +1000841 struct xfs_buf *bp;
842 DEFINE_SINGLE_BUF_MAP(map, XFS_BUF_DADDR_NULL, numblks);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700843
Dave Chinner3e85c862012-06-22 18:50:09 +1000844 bp = _xfs_buf_alloc(target, &map, 1, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700845 if (unlikely(bp == NULL))
846 goto fail;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700847
Dave Chinnere70b73f2012-04-23 15:58:49 +1000848 page_count = PAGE_ALIGN(numblks << BBSHIFT) >> PAGE_SHIFT;
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000849 error = _xfs_buf_get_pages(bp, page_count, 0);
850 if (error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700851 goto fail_free_buf;
852
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000853 for (i = 0; i < page_count; i++) {
Dave Chinner686865f2010-09-24 20:07:47 +1000854 bp->b_pages[i] = alloc_page(xb_to_gfp(flags));
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000855 if (!bp->b_pages[i])
856 goto fail_free_mem;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700857 }
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000858 bp->b_flags |= _XBF_PAGES;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700859
Dave Chinner611c9942012-04-23 15:59:07 +1000860 error = _xfs_buf_map_pages(bp, 0);
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000861 if (unlikely(error)) {
Dave Chinner4f107002011-03-07 10:00:35 +1100862 xfs_warn(target->bt_mount,
863 "%s: failed to map pages\n", __func__);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700864 goto fail_free_mem;
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000865 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700866
Dave Chinner686865f2010-09-24 20:07:47 +1000867 trace_xfs_buf_get_uncached(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700868 return bp;
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000869
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870 fail_free_mem:
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000871 while (--i >= 0)
872 __free_page(bp->b_pages[i]);
Christoph Hellwigca165b82007-05-24 15:21:11 +1000873 _xfs_buf_free_pages(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700874 fail_free_buf:
Dave Chinner3e85c862012-06-22 18:50:09 +1000875 xfs_buf_free_maps(bp);
Christoph Hellwig4347b9d2011-10-10 16:52:48 +0000876 kmem_zone_free(xfs_buf_zone, bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700877 fail:
878 return NULL;
879}
880
881/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700882 * Increment reference count on buffer, to hold the buffer concurrently
883 * with another thread which may release (free) the buffer asynchronously.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700884 * Must hold the buffer already to call this function.
885 */
886void
Nathan Scottce8e9222006-01-11 15:39:08 +1100887xfs_buf_hold(
888 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000890 trace_xfs_buf_hold(bp, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +1100891 atomic_inc(&bp->b_hold);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700892}
893
894/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100895 * Releases a hold on the specified buffer. If the
896 * the hold count is 1, calls xfs_buf_free.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700897 */
898void
Nathan Scottce8e9222006-01-11 15:39:08 +1100899xfs_buf_rele(
900 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901{
Dave Chinner74f75a02010-09-24 19:59:04 +1000902 struct xfs_perag *pag = bp->b_pag;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000904 trace_xfs_buf_rele(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700905
Dave Chinner74f75a02010-09-24 19:59:04 +1000906 if (!pag) {
Dave Chinner430cbeb2010-12-02 16:30:55 +1100907 ASSERT(list_empty(&bp->b_lru));
Dave Chinner74f75a02010-09-24 19:59:04 +1000908 ASSERT(RB_EMPTY_NODE(&bp->b_rbnode));
Nathan Scottfad3aa12006-02-01 12:14:52 +1100909 if (atomic_dec_and_test(&bp->b_hold))
910 xfs_buf_free(bp);
911 return;
912 }
913
Dave Chinner74f75a02010-09-24 19:59:04 +1000914 ASSERT(!RB_EMPTY_NODE(&bp->b_rbnode));
Dave Chinner430cbeb2010-12-02 16:30:55 +1100915
Lachlan McIlroy37906892008-08-13 15:42:10 +1000916 ASSERT(atomic_read(&bp->b_hold) > 0);
Dave Chinner74f75a02010-09-24 19:59:04 +1000917 if (atomic_dec_and_lock(&bp->b_hold, &pag->pag_buf_lock)) {
Christoph Hellwigbfc60172011-01-07 13:02:23 +0000918 if (!(bp->b_flags & XBF_STALE) &&
Dave Chinner430cbeb2010-12-02 16:30:55 +1100919 atomic_read(&bp->b_lru_ref)) {
920 xfs_buf_lru_add(bp);
921 spin_unlock(&pag->pag_buf_lock);
Christoph Hellwig7f14d0a2005-11-02 15:09:35 +1100922 } else {
Dave Chinner430cbeb2010-12-02 16:30:55 +1100923 xfs_buf_lru_del(bp);
Christoph Hellwig43ff2122012-04-23 15:58:39 +1000924 ASSERT(!(bp->b_flags & _XBF_DELWRI_Q));
Dave Chinner74f75a02010-09-24 19:59:04 +1000925 rb_erase(&bp->b_rbnode, &pag->pag_buf_tree);
926 spin_unlock(&pag->pag_buf_lock);
927 xfs_perag_put(pag);
Nathan Scottce8e9222006-01-11 15:39:08 +1100928 xfs_buf_free(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929 }
930 }
931}
932
933
934/*
Dave Chinner0e6e8472011-03-26 09:16:45 +1100935 * Lock a buffer object, if it is not already locked.
Dave Chinner90810b92010-11-30 15:16:16 +1100936 *
937 * If we come across a stale, pinned, locked buffer, we know that we are
938 * being asked to lock a buffer that has been reallocated. Because it is
939 * pinned, we know that the log has not been pushed to disk and hence it
940 * will still be locked. Rather than continuing to have trylock attempts
941 * fail until someone else pushes the log, push it ourselves before
942 * returning. This means that the xfsaild will not get stuck trying
943 * to push on stale inode buffers.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700944 */
945int
Christoph Hellwig0c842ad2011-07-08 14:36:19 +0200946xfs_buf_trylock(
947 struct xfs_buf *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700948{
949 int locked;
950
Nathan Scottce8e9222006-01-11 15:39:08 +1100951 locked = down_trylock(&bp->b_sema) == 0;
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000952 if (locked)
Nathan Scottce8e9222006-01-11 15:39:08 +1100953 XB_SET_OWNER(bp);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000954
Christoph Hellwig0c842ad2011-07-08 14:36:19 +0200955 trace_xfs_buf_trylock(bp, _RET_IP_);
956 return locked;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700957}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958
959/*
Dave Chinner0e6e8472011-03-26 09:16:45 +1100960 * Lock a buffer object.
Dave Chinnered3b4d62010-05-21 12:07:08 +1000961 *
962 * If we come across a stale, pinned, locked buffer, we know that we
963 * are being asked to lock a buffer that has been reallocated. Because
964 * it is pinned, we know that the log has not been pushed to disk and
965 * hence it will still be locked. Rather than sleeping until someone
966 * else pushes the log, push it ourselves before trying to get the lock.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700967 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100968void
969xfs_buf_lock(
Christoph Hellwig0c842ad2011-07-08 14:36:19 +0200970 struct xfs_buf *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700971{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000972 trace_xfs_buf_lock(bp, _RET_IP_);
973
Dave Chinnered3b4d62010-05-21 12:07:08 +1000974 if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE))
Dave Chinnerebad8612010-09-22 10:47:20 +1000975 xfs_log_force(bp->b_target->bt_mount, 0);
Nathan Scottce8e9222006-01-11 15:39:08 +1100976 down(&bp->b_sema);
977 XB_SET_OWNER(bp);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000978
979 trace_xfs_buf_lock_done(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700980}
981
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982void
Nathan Scottce8e9222006-01-11 15:39:08 +1100983xfs_buf_unlock(
Christoph Hellwig0c842ad2011-07-08 14:36:19 +0200984 struct xfs_buf *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700985{
Nathan Scottce8e9222006-01-11 15:39:08 +1100986 XB_CLEAR_OWNER(bp);
987 up(&bp->b_sema);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000988
989 trace_xfs_buf_unlock(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700990}
991
Nathan Scottce8e9222006-01-11 15:39:08 +1100992STATIC void
993xfs_buf_wait_unpin(
994 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700995{
996 DECLARE_WAITQUEUE (wait, current);
997
Nathan Scottce8e9222006-01-11 15:39:08 +1100998 if (atomic_read(&bp->b_pin_count) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700999 return;
1000
Nathan Scottce8e9222006-01-11 15:39:08 +11001001 add_wait_queue(&bp->b_waiters, &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001002 for (;;) {
1003 set_current_state(TASK_UNINTERRUPTIBLE);
Nathan Scottce8e9222006-01-11 15:39:08 +11001004 if (atomic_read(&bp->b_pin_count) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001005 break;
Jens Axboe7eaceac2011-03-10 08:52:07 +01001006 io_schedule();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001007 }
Nathan Scottce8e9222006-01-11 15:39:08 +11001008 remove_wait_queue(&bp->b_waiters, &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001009 set_current_state(TASK_RUNNING);
1010}
1011
1012/*
1013 * Buffer Utility Routines
1014 */
1015
Linus Torvalds1da177e2005-04-16 15:20:36 -07001016STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001017xfs_buf_iodone_work(
David Howellsc4028952006-11-22 14:57:56 +00001018 struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001019{
Dave Chinner1813dd62012-11-14 17:54:40 +11001020 struct xfs_buf *bp =
David Howellsc4028952006-11-22 14:57:56 +00001021 container_of(work, xfs_buf_t, b_iodone_work);
Dave Chinner1813dd62012-11-14 17:54:40 +11001022 bool read = !!(bp->b_flags & XBF_READ);
1023
1024 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
1025 if (read && bp->b_ops)
1026 bp->b_ops->verify_read(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001027
Christoph Hellwig80f6c292010-08-18 05:29:11 -04001028 if (bp->b_iodone)
Nathan Scottce8e9222006-01-11 15:39:08 +11001029 (*(bp->b_iodone))(bp);
1030 else if (bp->b_flags & XBF_ASYNC)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031 xfs_buf_relse(bp);
Dave Chinner1813dd62012-11-14 17:54:40 +11001032 else {
1033 ASSERT(read && bp->b_ops);
1034 complete(&bp->b_iowait);
1035 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001036}
1037
1038void
Nathan Scottce8e9222006-01-11 15:39:08 +11001039xfs_buf_ioend(
Dave Chinner1813dd62012-11-14 17:54:40 +11001040 struct xfs_buf *bp,
1041 int schedule)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001042{
Dave Chinner1813dd62012-11-14 17:54:40 +11001043 bool read = !!(bp->b_flags & XBF_READ);
1044
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001045 trace_xfs_buf_iodone(bp, _RET_IP_);
1046
Nathan Scottce8e9222006-01-11 15:39:08 +11001047 if (bp->b_error == 0)
1048 bp->b_flags |= XBF_DONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001049
Dave Chinner1813dd62012-11-14 17:54:40 +11001050 if (bp->b_iodone || (read && bp->b_ops) || (bp->b_flags & XBF_ASYNC)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001051 if (schedule) {
David Howellsc4028952006-11-22 14:57:56 +00001052 INIT_WORK(&bp->b_iodone_work, xfs_buf_iodone_work);
Nathan Scottce8e9222006-01-11 15:39:08 +11001053 queue_work(xfslogd_workqueue, &bp->b_iodone_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001054 } else {
David Howellsc4028952006-11-22 14:57:56 +00001055 xfs_buf_iodone_work(&bp->b_iodone_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001056 }
1057 } else {
Dave Chinner1813dd62012-11-14 17:54:40 +11001058 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
David Chinnerb4dd3302008-08-13 16:36:11 +10001059 complete(&bp->b_iowait);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001060 }
1061}
1062
Linus Torvalds1da177e2005-04-16 15:20:36 -07001063void
Nathan Scottce8e9222006-01-11 15:39:08 +11001064xfs_buf_ioerror(
1065 xfs_buf_t *bp,
1066 int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001067{
1068 ASSERT(error >= 0 && error <= 0xffff);
Nathan Scottce8e9222006-01-11 15:39:08 +11001069 bp->b_error = (unsigned short)error;
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001070 trace_xfs_buf_ioerror(bp, error, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001071}
1072
Christoph Hellwig901796a2011-10-10 16:52:49 +00001073void
1074xfs_buf_ioerror_alert(
1075 struct xfs_buf *bp,
1076 const char *func)
1077{
1078 xfs_alert(bp->b_target->bt_mount,
Dave Chinneraa0e8832012-04-23 15:58:52 +10001079"metadata I/O error: block 0x%llx (\"%s\") error %d numblks %d",
1080 (__uint64_t)XFS_BUF_ADDR(bp), func, bp->b_error, bp->b_length);
Christoph Hellwig901796a2011-10-10 16:52:49 +00001081}
1082
Christoph Hellwig4e234712010-01-13 22:17:56 +00001083/*
1084 * Called when we want to stop a buffer from getting written or read.
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +00001085 * We attach the EIO error, muck with its flags, and call xfs_buf_ioend
Christoph Hellwig4e234712010-01-13 22:17:56 +00001086 * so that the proper iodone callbacks get called.
1087 */
1088STATIC int
1089xfs_bioerror(
1090 xfs_buf_t *bp)
1091{
1092#ifdef XFSERRORDEBUG
1093 ASSERT(XFS_BUF_ISREAD(bp) || bp->b_iodone);
1094#endif
1095
1096 /*
1097 * No need to wait until the buffer is unpinned, we aren't flushing it.
1098 */
Chandra Seetharaman5a52c2a582011-07-22 23:39:51 +00001099 xfs_buf_ioerror(bp, EIO);
Christoph Hellwig4e234712010-01-13 22:17:56 +00001100
1101 /*
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +00001102 * We're calling xfs_buf_ioend, so delete XBF_DONE flag.
Christoph Hellwig4e234712010-01-13 22:17:56 +00001103 */
1104 XFS_BUF_UNREAD(bp);
Christoph Hellwig4e234712010-01-13 22:17:56 +00001105 XFS_BUF_UNDONE(bp);
Christoph Hellwigc867cb62011-10-10 16:52:46 +00001106 xfs_buf_stale(bp);
Christoph Hellwig4e234712010-01-13 22:17:56 +00001107
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +00001108 xfs_buf_ioend(bp, 0);
Christoph Hellwig4e234712010-01-13 22:17:56 +00001109
1110 return EIO;
1111}
1112
1113/*
1114 * Same as xfs_bioerror, except that we are releasing the buffer
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +00001115 * here ourselves, and avoiding the xfs_buf_ioend call.
Christoph Hellwig4e234712010-01-13 22:17:56 +00001116 * This is meant for userdata errors; metadata bufs come with
1117 * iodone functions attached, so that we can track down errors.
1118 */
1119STATIC int
1120xfs_bioerror_relse(
1121 struct xfs_buf *bp)
1122{
Chandra Seetharamaned432332011-07-22 23:39:39 +00001123 int64_t fl = bp->b_flags;
Christoph Hellwig4e234712010-01-13 22:17:56 +00001124 /*
1125 * No need to wait until the buffer is unpinned.
1126 * We aren't flushing it.
1127 *
1128 * chunkhold expects B_DONE to be set, whether
1129 * we actually finish the I/O or not. We don't want to
1130 * change that interface.
1131 */
1132 XFS_BUF_UNREAD(bp);
Christoph Hellwig4e234712010-01-13 22:17:56 +00001133 XFS_BUF_DONE(bp);
Christoph Hellwigc867cb62011-10-10 16:52:46 +00001134 xfs_buf_stale(bp);
Christoph Hellwigcb669ca2011-07-13 13:43:49 +02001135 bp->b_iodone = NULL;
Christoph Hellwig0cadda12010-01-19 09:56:44 +00001136 if (!(fl & XBF_ASYNC)) {
Christoph Hellwig4e234712010-01-13 22:17:56 +00001137 /*
1138 * Mark b_error and B_ERROR _both_.
1139 * Lot's of chunkcache code assumes that.
1140 * There's no reason to mark error for
1141 * ASYNC buffers.
1142 */
Chandra Seetharaman5a52c2a582011-07-22 23:39:51 +00001143 xfs_buf_ioerror(bp, EIO);
Christoph Hellwig5fde0322011-10-10 16:52:44 +00001144 complete(&bp->b_iowait);
Christoph Hellwig4e234712010-01-13 22:17:56 +00001145 } else {
1146 xfs_buf_relse(bp);
1147 }
1148
1149 return EIO;
1150}
1151
Christoph Hellwiga2dcf5d2012-07-13 02:24:10 -04001152STATIC int
Christoph Hellwig4e234712010-01-13 22:17:56 +00001153xfs_bdstrat_cb(
1154 struct xfs_buf *bp)
1155{
Dave Chinnerebad8612010-09-22 10:47:20 +10001156 if (XFS_FORCED_SHUTDOWN(bp->b_target->bt_mount)) {
Christoph Hellwig4e234712010-01-13 22:17:56 +00001157 trace_xfs_bdstrat_shut(bp, _RET_IP_);
1158 /*
1159 * Metadata write that didn't get logged but
1160 * written delayed anyway. These aren't associated
1161 * with a transaction, and can be ignored.
1162 */
1163 if (!bp->b_iodone && !XFS_BUF_ISREAD(bp))
1164 return xfs_bioerror_relse(bp);
1165 else
1166 return xfs_bioerror(bp);
1167 }
1168
1169 xfs_buf_iorequest(bp);
1170 return 0;
1171}
1172
Christoph Hellwiga2dcf5d2012-07-13 02:24:10 -04001173int
1174xfs_bwrite(
1175 struct xfs_buf *bp)
1176{
1177 int error;
1178
1179 ASSERT(xfs_buf_islocked(bp));
1180
1181 bp->b_flags |= XBF_WRITE;
1182 bp->b_flags &= ~(XBF_ASYNC | XBF_READ | _XBF_DELWRI_Q);
1183
1184 xfs_bdstrat_cb(bp);
1185
1186 error = xfs_buf_iowait(bp);
1187 if (error) {
1188 xfs_force_shutdown(bp->b_target->bt_mount,
1189 SHUTDOWN_META_IO_ERROR);
1190 }
1191 return error;
1192}
1193
Christoph Hellwig4e234712010-01-13 22:17:56 +00001194/*
1195 * Wrapper around bdstrat so that we can stop data from going to disk in case
1196 * we are shutting down the filesystem. Typically user data goes thru this
1197 * path; one of the exceptions is the superblock.
1198 */
1199void
1200xfsbdstrat(
1201 struct xfs_mount *mp,
1202 struct xfs_buf *bp)
1203{
1204 if (XFS_FORCED_SHUTDOWN(mp)) {
1205 trace_xfs_bdstrat_shut(bp, _RET_IP_);
1206 xfs_bioerror_relse(bp);
1207 return;
1208 }
1209
1210 xfs_buf_iorequest(bp);
1211}
1212
Christoph Hellwigb8f82a42009-11-14 16:17:22 +00001213STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001214_xfs_buf_ioend(
1215 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001216 int schedule)
1217{
Dave Chinner0e6e8472011-03-26 09:16:45 +11001218 if (atomic_dec_and_test(&bp->b_io_remaining) == 1)
Nathan Scottce8e9222006-01-11 15:39:08 +11001219 xfs_buf_ioend(bp, schedule);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001220}
1221
Al Viro782e3b32007-10-12 07:17:47 +01001222STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001223xfs_buf_bio_end_io(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001224 struct bio *bio,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001225 int error)
1226{
Nathan Scottce8e9222006-01-11 15:39:08 +11001227 xfs_buf_t *bp = (xfs_buf_t *)bio->bi_private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001228
Dave Chinner37eb17e2012-11-12 22:09:46 +11001229 /*
1230 * don't overwrite existing errors - otherwise we can lose errors on
1231 * buffers that require multiple bios to complete.
1232 */
1233 if (!bp->b_error)
1234 xfs_buf_ioerror(bp, -error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001235
Dave Chinner37eb17e2012-11-12 22:09:46 +11001236 if (!bp->b_error && xfs_buf_is_vmapped(bp) && (bp->b_flags & XBF_READ))
James Bottomley73c77e22010-01-25 11:42:24 -06001237 invalidate_kernel_vmap_range(bp->b_addr, xfs_buf_vmap_len(bp));
1238
Nathan Scottce8e9222006-01-11 15:39:08 +11001239 _xfs_buf_ioend(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001240 bio_put(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001241}
1242
Dave Chinner3e85c862012-06-22 18:50:09 +10001243static void
1244xfs_buf_ioapply_map(
1245 struct xfs_buf *bp,
1246 int map,
1247 int *buf_offset,
1248 int *count,
1249 int rw)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001250{
Dave Chinner3e85c862012-06-22 18:50:09 +10001251 int page_index;
1252 int total_nr_pages = bp->b_page_count;
1253 int nr_pages;
1254 struct bio *bio;
1255 sector_t sector = bp->b_maps[map].bm_bn;
1256 int size;
1257 int offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001258
Nathan Scottce8e9222006-01-11 15:39:08 +11001259 total_nr_pages = bp->b_page_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001260
Dave Chinner3e85c862012-06-22 18:50:09 +10001261 /* skip the pages in the buffer before the start offset */
1262 page_index = 0;
1263 offset = *buf_offset;
1264 while (offset >= PAGE_SIZE) {
1265 page_index++;
1266 offset -= PAGE_SIZE;
Christoph Hellwigf538d4d2005-11-02 10:26:59 +11001267 }
1268
Dave Chinner3e85c862012-06-22 18:50:09 +10001269 /*
1270 * Limit the IO size to the length of the current vector, and update the
1271 * remaining IO count for the next time around.
1272 */
1273 size = min_t(int, BBTOB(bp->b_maps[map].bm_len), *count);
1274 *count -= size;
1275 *buf_offset += size;
Christoph Hellwig34951f52011-07-26 15:06:44 +00001276
Linus Torvalds1da177e2005-04-16 15:20:36 -07001277next_chunk:
Nathan Scottce8e9222006-01-11 15:39:08 +11001278 atomic_inc(&bp->b_io_remaining);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001279 nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1280 if (nr_pages > total_nr_pages)
1281 nr_pages = total_nr_pages;
1282
1283 bio = bio_alloc(GFP_NOIO, nr_pages);
Nathan Scottce8e9222006-01-11 15:39:08 +11001284 bio->bi_bdev = bp->b_target->bt_bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001285 bio->bi_sector = sector;
Nathan Scottce8e9222006-01-11 15:39:08 +11001286 bio->bi_end_io = xfs_buf_bio_end_io;
1287 bio->bi_private = bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001288
Dave Chinner0e6e8472011-03-26 09:16:45 +11001289
Dave Chinner3e85c862012-06-22 18:50:09 +10001290 for (; size && nr_pages; nr_pages--, page_index++) {
Dave Chinner0e6e8472011-03-26 09:16:45 +11001291 int rbytes, nbytes = PAGE_SIZE - offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001292
1293 if (nbytes > size)
1294 nbytes = size;
1295
Dave Chinner3e85c862012-06-22 18:50:09 +10001296 rbytes = bio_add_page(bio, bp->b_pages[page_index], nbytes,
1297 offset);
Nathan Scottce8e9222006-01-11 15:39:08 +11001298 if (rbytes < nbytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001299 break;
1300
1301 offset = 0;
Dave Chinneraa0e8832012-04-23 15:58:52 +10001302 sector += BTOBB(nbytes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001303 size -= nbytes;
1304 total_nr_pages--;
1305 }
1306
Linus Torvalds1da177e2005-04-16 15:20:36 -07001307 if (likely(bio->bi_size)) {
James Bottomley73c77e22010-01-25 11:42:24 -06001308 if (xfs_buf_is_vmapped(bp)) {
1309 flush_kernel_vmap_range(bp->b_addr,
1310 xfs_buf_vmap_len(bp));
1311 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001312 submit_bio(rw, bio);
1313 if (size)
1314 goto next_chunk;
1315 } else {
Dave Chinner37eb17e2012-11-12 22:09:46 +11001316 /*
1317 * This is guaranteed not to be the last io reference count
1318 * because the caller (xfs_buf_iorequest) holds a count itself.
1319 */
1320 atomic_dec(&bp->b_io_remaining);
Nathan Scottce8e9222006-01-11 15:39:08 +11001321 xfs_buf_ioerror(bp, EIO);
Dave Chinnerec53d1d2010-07-20 17:52:59 +10001322 bio_put(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001323 }
Dave Chinner3e85c862012-06-22 18:50:09 +10001324
1325}
1326
1327STATIC void
1328_xfs_buf_ioapply(
1329 struct xfs_buf *bp)
1330{
1331 struct blk_plug plug;
1332 int rw;
1333 int offset;
1334 int size;
1335 int i;
1336
1337 if (bp->b_flags & XBF_WRITE) {
1338 if (bp->b_flags & XBF_SYNCIO)
1339 rw = WRITE_SYNC;
1340 else
1341 rw = WRITE;
1342 if (bp->b_flags & XBF_FUA)
1343 rw |= REQ_FUA;
1344 if (bp->b_flags & XBF_FLUSH)
1345 rw |= REQ_FLUSH;
Dave Chinner1813dd62012-11-14 17:54:40 +11001346
1347 /*
1348 * Run the write verifier callback function if it exists. If
1349 * this function fails it will mark the buffer with an error and
1350 * the IO should not be dispatched.
1351 */
1352 if (bp->b_ops) {
1353 bp->b_ops->verify_write(bp);
1354 if (bp->b_error) {
1355 xfs_force_shutdown(bp->b_target->bt_mount,
1356 SHUTDOWN_CORRUPT_INCORE);
1357 return;
1358 }
1359 }
Dave Chinner3e85c862012-06-22 18:50:09 +10001360 } else if (bp->b_flags & XBF_READ_AHEAD) {
1361 rw = READA;
1362 } else {
1363 rw = READ;
1364 }
1365
1366 /* we only use the buffer cache for meta-data */
1367 rw |= REQ_META;
1368
1369 /*
1370 * Walk all the vectors issuing IO on them. Set up the initial offset
1371 * into the buffer and the desired IO size before we start -
1372 * _xfs_buf_ioapply_vec() will modify them appropriately for each
1373 * subsequent call.
1374 */
1375 offset = bp->b_offset;
1376 size = BBTOB(bp->b_io_length);
1377 blk_start_plug(&plug);
1378 for (i = 0; i < bp->b_map_count; i++) {
1379 xfs_buf_ioapply_map(bp, i, &offset, &size, rw);
1380 if (bp->b_error)
1381 break;
1382 if (size <= 0)
1383 break; /* all done */
1384 }
1385 blk_finish_plug(&plug);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001386}
1387
Dave Chinner0e95f192012-04-23 15:58:46 +10001388void
Nathan Scottce8e9222006-01-11 15:39:08 +11001389xfs_buf_iorequest(
1390 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001391{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001392 trace_xfs_buf_iorequest(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001393
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001394 ASSERT(!(bp->b_flags & _XBF_DELWRI_Q));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001395
Christoph Hellwig375ec69d2011-08-23 08:28:03 +00001396 if (bp->b_flags & XBF_WRITE)
Nathan Scottce8e9222006-01-11 15:39:08 +11001397 xfs_buf_wait_unpin(bp);
Nathan Scottce8e9222006-01-11 15:39:08 +11001398 xfs_buf_hold(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001399
1400 /* Set the count to 1 initially, this will stop an I/O
1401 * completion callout which happens before we have started
Nathan Scottce8e9222006-01-11 15:39:08 +11001402 * all the I/O from calling xfs_buf_ioend too early.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001403 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001404 atomic_set(&bp->b_io_remaining, 1);
1405 _xfs_buf_ioapply(bp);
Christoph Hellwig08023d62012-07-02 06:00:04 -04001406 _xfs_buf_ioend(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001407
Nathan Scottce8e9222006-01-11 15:39:08 +11001408 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001409}
1410
1411/*
Dave Chinner0e95f192012-04-23 15:58:46 +10001412 * Waits for I/O to complete on the buffer supplied. It returns immediately if
1413 * no I/O is pending or there is already a pending error on the buffer. It
1414 * returns the I/O error code, if any, or 0 if there was no error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001415 */
1416int
Nathan Scottce8e9222006-01-11 15:39:08 +11001417xfs_buf_iowait(
1418 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001419{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001420 trace_xfs_buf_iowait(bp, _RET_IP_);
1421
Dave Chinner0e95f192012-04-23 15:58:46 +10001422 if (!bp->b_error)
1423 wait_for_completion(&bp->b_iowait);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001424
1425 trace_xfs_buf_iowait_done(bp, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +11001426 return bp->b_error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001427}
1428
Nathan Scottce8e9222006-01-11 15:39:08 +11001429xfs_caddr_t
1430xfs_buf_offset(
1431 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001432 size_t offset)
1433{
1434 struct page *page;
1435
Dave Chinner611c9942012-04-23 15:59:07 +10001436 if (bp->b_addr)
Chandra Seetharaman62926042011-07-22 23:40:15 +00001437 return bp->b_addr + offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001438
Nathan Scottce8e9222006-01-11 15:39:08 +11001439 offset += bp->b_offset;
Dave Chinner0e6e8472011-03-26 09:16:45 +11001440 page = bp->b_pages[offset >> PAGE_SHIFT];
1441 return (xfs_caddr_t)page_address(page) + (offset & (PAGE_SIZE-1));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001442}
1443
1444/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001445 * Move data into or out of a buffer.
1446 */
1447void
Nathan Scottce8e9222006-01-11 15:39:08 +11001448xfs_buf_iomove(
1449 xfs_buf_t *bp, /* buffer to process */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001450 size_t boff, /* starting buffer offset */
1451 size_t bsize, /* length to copy */
Dave Chinnerb9c48642010-01-20 10:47:39 +11001452 void *data, /* data address */
Nathan Scottce8e9222006-01-11 15:39:08 +11001453 xfs_buf_rw_t mode) /* read/write/zero flag */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001454{
Dave Chinner795cac72012-04-23 15:58:53 +10001455 size_t bend;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001456
1457 bend = boff + bsize;
1458 while (boff < bend) {
Dave Chinner795cac72012-04-23 15:58:53 +10001459 struct page *page;
1460 int page_index, page_offset, csize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001461
Dave Chinner795cac72012-04-23 15:58:53 +10001462 page_index = (boff + bp->b_offset) >> PAGE_SHIFT;
1463 page_offset = (boff + bp->b_offset) & ~PAGE_MASK;
1464 page = bp->b_pages[page_index];
1465 csize = min_t(size_t, PAGE_SIZE - page_offset,
1466 BBTOB(bp->b_io_length) - boff);
1467
1468 ASSERT((csize + page_offset) <= PAGE_SIZE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001469
1470 switch (mode) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001471 case XBRW_ZERO:
Dave Chinner795cac72012-04-23 15:58:53 +10001472 memset(page_address(page) + page_offset, 0, csize);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001473 break;
Nathan Scottce8e9222006-01-11 15:39:08 +11001474 case XBRW_READ:
Dave Chinner795cac72012-04-23 15:58:53 +10001475 memcpy(data, page_address(page) + page_offset, csize);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001476 break;
Nathan Scottce8e9222006-01-11 15:39:08 +11001477 case XBRW_WRITE:
Dave Chinner795cac72012-04-23 15:58:53 +10001478 memcpy(page_address(page) + page_offset, data, csize);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001479 }
1480
1481 boff += csize;
1482 data += csize;
1483 }
1484}
1485
1486/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001487 * Handling of buffer targets (buftargs).
Linus Torvalds1da177e2005-04-16 15:20:36 -07001488 */
1489
1490/*
Dave Chinner430cbeb2010-12-02 16:30:55 +11001491 * Wait for any bufs with callbacks that have been submitted but have not yet
1492 * returned. These buffers will have an elevated hold count, so wait on those
1493 * while freeing all the buffers only held by the LRU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001494 */
1495void
1496xfs_wait_buftarg(
Dave Chinner74f75a02010-09-24 19:59:04 +10001497 struct xfs_buftarg *btp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001498{
Dave Chinner430cbeb2010-12-02 16:30:55 +11001499 struct xfs_buf *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001500
Dave Chinner430cbeb2010-12-02 16:30:55 +11001501restart:
1502 spin_lock(&btp->bt_lru_lock);
1503 while (!list_empty(&btp->bt_lru)) {
1504 bp = list_first_entry(&btp->bt_lru, struct xfs_buf, b_lru);
1505 if (atomic_read(&bp->b_hold) > 1) {
Dave Chinner3b190342013-01-21 23:53:55 +11001506 trace_xfs_buf_wait_buftarg(bp, _RET_IP_);
1507 list_move_tail(&bp->b_lru, &btp->bt_lru);
Dave Chinner430cbeb2010-12-02 16:30:55 +11001508 spin_unlock(&btp->bt_lru_lock);
Dave Chinner26af6552010-09-22 10:47:20 +10001509 delay(100);
Dave Chinner430cbeb2010-12-02 16:30:55 +11001510 goto restart;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001511 }
Dave Chinner430cbeb2010-12-02 16:30:55 +11001512 /*
Paul Bolle90802ed2011-12-05 13:00:34 +01001513 * clear the LRU reference count so the buffer doesn't get
Dave Chinner430cbeb2010-12-02 16:30:55 +11001514 * ignored in xfs_buf_rele().
1515 */
1516 atomic_set(&bp->b_lru_ref, 0);
1517 spin_unlock(&btp->bt_lru_lock);
1518 xfs_buf_rele(bp);
1519 spin_lock(&btp->bt_lru_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001520 }
Dave Chinner430cbeb2010-12-02 16:30:55 +11001521 spin_unlock(&btp->bt_lru_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001522}
1523
Dave Chinnerff57ab22010-11-30 17:27:57 +11001524int
1525xfs_buftarg_shrink(
1526 struct shrinker *shrink,
Ying Han1495f232011-05-24 17:12:27 -07001527 struct shrink_control *sc)
David Chinnera6867a62006-01-11 15:37:58 +11001528{
Dave Chinnerff57ab22010-11-30 17:27:57 +11001529 struct xfs_buftarg *btp = container_of(shrink,
1530 struct xfs_buftarg, bt_shrinker);
Dave Chinner430cbeb2010-12-02 16:30:55 +11001531 struct xfs_buf *bp;
Ying Han1495f232011-05-24 17:12:27 -07001532 int nr_to_scan = sc->nr_to_scan;
Dave Chinner430cbeb2010-12-02 16:30:55 +11001533 LIST_HEAD(dispose);
1534
1535 if (!nr_to_scan)
1536 return btp->bt_lru_nr;
1537
1538 spin_lock(&btp->bt_lru_lock);
1539 while (!list_empty(&btp->bt_lru)) {
1540 if (nr_to_scan-- <= 0)
1541 break;
1542
1543 bp = list_first_entry(&btp->bt_lru, struct xfs_buf, b_lru);
1544
1545 /*
1546 * Decrement the b_lru_ref count unless the value is already
1547 * zero. If the value is already zero, we need to reclaim the
1548 * buffer, otherwise it gets another trip through the LRU.
1549 */
1550 if (!atomic_add_unless(&bp->b_lru_ref, -1, 0)) {
1551 list_move_tail(&bp->b_lru, &btp->bt_lru);
1552 continue;
1553 }
1554
1555 /*
1556 * remove the buffer from the LRU now to avoid needing another
1557 * lock round trip inside xfs_buf_rele().
1558 */
1559 list_move(&bp->b_lru, &dispose);
1560 btp->bt_lru_nr--;
Carlos Maiolino6fb8a902012-08-10 15:01:51 -03001561 bp->b_lru_flags |= _XBF_LRU_DISPOSE;
Dave Chinnerff57ab22010-11-30 17:27:57 +11001562 }
Dave Chinner430cbeb2010-12-02 16:30:55 +11001563 spin_unlock(&btp->bt_lru_lock);
1564
1565 while (!list_empty(&dispose)) {
1566 bp = list_first_entry(&dispose, struct xfs_buf, b_lru);
1567 list_del_init(&bp->b_lru);
1568 xfs_buf_rele(bp);
1569 }
1570
1571 return btp->bt_lru_nr;
David Chinnera6867a62006-01-11 15:37:58 +11001572}
1573
Linus Torvalds1da177e2005-04-16 15:20:36 -07001574void
1575xfs_free_buftarg(
Christoph Hellwigb7963132009-03-03 14:48:37 -05001576 struct xfs_mount *mp,
1577 struct xfs_buftarg *btp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001578{
Dave Chinnerff57ab22010-11-30 17:27:57 +11001579 unregister_shrinker(&btp->bt_shrinker);
1580
Christoph Hellwigb7963132009-03-03 14:48:37 -05001581 if (mp->m_flags & XFS_MOUNT_BARRIER)
1582 xfs_blkdev_issue_flush(btp);
David Chinnera6867a62006-01-11 15:37:58 +11001583
Denys Vlasenkof0e2d932008-05-19 16:31:57 +10001584 kmem_free(btp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001585}
1586
Linus Torvalds1da177e2005-04-16 15:20:36 -07001587STATIC int
1588xfs_setsize_buftarg_flags(
1589 xfs_buftarg_t *btp,
1590 unsigned int blocksize,
1591 unsigned int sectorsize,
1592 int verbose)
1593{
Nathan Scottce8e9222006-01-11 15:39:08 +11001594 btp->bt_bsize = blocksize;
1595 btp->bt_sshift = ffs(sectorsize) - 1;
1596 btp->bt_smask = sectorsize - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001597
Nathan Scottce8e9222006-01-11 15:39:08 +11001598 if (set_blocksize(btp->bt_bdev, sectorsize)) {
Christoph Hellwig02b102d2011-10-10 16:52:51 +00001599 char name[BDEVNAME_SIZE];
1600
1601 bdevname(btp->bt_bdev, name);
1602
Dave Chinner4f107002011-03-07 10:00:35 +11001603 xfs_warn(btp->bt_mount,
1604 "Cannot set_blocksize to %u on device %s\n",
Christoph Hellwig02b102d2011-10-10 16:52:51 +00001605 sectorsize, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001606 return EINVAL;
1607 }
1608
Linus Torvalds1da177e2005-04-16 15:20:36 -07001609 return 0;
1610}
1611
1612/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001613 * When allocating the initial buffer target we have not yet
1614 * read in the superblock, so don't know what sized sectors
1615 * are being used is at this early stage. Play safe.
1616 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001617STATIC int
1618xfs_setsize_buftarg_early(
1619 xfs_buftarg_t *btp,
1620 struct block_device *bdev)
1621{
1622 return xfs_setsize_buftarg_flags(btp,
Dave Chinner0e6e8472011-03-26 09:16:45 +11001623 PAGE_SIZE, bdev_logical_block_size(bdev), 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001624}
1625
1626int
1627xfs_setsize_buftarg(
1628 xfs_buftarg_t *btp,
1629 unsigned int blocksize,
1630 unsigned int sectorsize)
1631{
1632 return xfs_setsize_buftarg_flags(btp, blocksize, sectorsize, 1);
1633}
1634
Linus Torvalds1da177e2005-04-16 15:20:36 -07001635xfs_buftarg_t *
1636xfs_alloc_buftarg(
Dave Chinnerebad8612010-09-22 10:47:20 +10001637 struct xfs_mount *mp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001638 struct block_device *bdev,
Jan Engelhardte2a07812010-03-23 09:52:55 +11001639 int external,
1640 const char *fsname)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001641{
1642 xfs_buftarg_t *btp;
1643
1644 btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
1645
Dave Chinnerebad8612010-09-22 10:47:20 +10001646 btp->bt_mount = mp;
Nathan Scottce8e9222006-01-11 15:39:08 +11001647 btp->bt_dev = bdev->bd_dev;
1648 btp->bt_bdev = bdev;
Dave Chinner0e6e8472011-03-26 09:16:45 +11001649 btp->bt_bdi = blk_get_backing_dev_info(bdev);
1650 if (!btp->bt_bdi)
1651 goto error;
1652
Dave Chinner430cbeb2010-12-02 16:30:55 +11001653 INIT_LIST_HEAD(&btp->bt_lru);
1654 spin_lock_init(&btp->bt_lru_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001655 if (xfs_setsize_buftarg_early(btp, bdev))
1656 goto error;
Dave Chinnerff57ab22010-11-30 17:27:57 +11001657 btp->bt_shrinker.shrink = xfs_buftarg_shrink;
1658 btp->bt_shrinker.seeks = DEFAULT_SEEKS;
1659 register_shrinker(&btp->bt_shrinker);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001660 return btp;
1661
1662error:
Denys Vlasenkof0e2d932008-05-19 16:31:57 +10001663 kmem_free(btp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001664 return NULL;
1665}
1666
Linus Torvalds1da177e2005-04-16 15:20:36 -07001667/*
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001668 * Add a buffer to the delayed write list.
1669 *
1670 * This queues a buffer for writeout if it hasn't already been. Note that
1671 * neither this routine nor the buffer list submission functions perform
1672 * any internal synchronization. It is expected that the lists are thread-local
1673 * to the callers.
1674 *
1675 * Returns true if we queued up the buffer, or false if it already had
1676 * been on the buffer list.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001677 */
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001678bool
Nathan Scottce8e9222006-01-11 15:39:08 +11001679xfs_buf_delwri_queue(
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001680 struct xfs_buf *bp,
1681 struct list_head *list)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001682{
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001683 ASSERT(xfs_buf_islocked(bp));
1684 ASSERT(!(bp->b_flags & XBF_READ));
1685
1686 /*
1687 * If the buffer is already marked delwri it already is queued up
1688 * by someone else for imediate writeout. Just ignore it in that
1689 * case.
1690 */
1691 if (bp->b_flags & _XBF_DELWRI_Q) {
1692 trace_xfs_buf_delwri_queued(bp, _RET_IP_);
1693 return false;
1694 }
David Chinnera6867a62006-01-11 15:37:58 +11001695
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001696 trace_xfs_buf_delwri_queue(bp, _RET_IP_);
1697
Dave Chinnerd808f612010-02-02 10:13:42 +11001698 /*
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001699 * If a buffer gets written out synchronously or marked stale while it
1700 * is on a delwri list we lazily remove it. To do this, the other party
1701 * clears the _XBF_DELWRI_Q flag but otherwise leaves the buffer alone.
1702 * It remains referenced and on the list. In a rare corner case it
1703 * might get readded to a delwri list after the synchronous writeout, in
1704 * which case we need just need to re-add the flag here.
Dave Chinnerd808f612010-02-02 10:13:42 +11001705 */
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001706 bp->b_flags |= _XBF_DELWRI_Q;
1707 if (list_empty(&bp->b_list)) {
1708 atomic_inc(&bp->b_hold);
1709 list_add_tail(&bp->b_list, list);
David Chinner585e6d82007-02-10 18:32:29 +11001710 }
David Chinner585e6d82007-02-10 18:32:29 +11001711
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001712 return true;
David Chinner585e6d82007-02-10 18:32:29 +11001713}
1714
Dave Chinner089716a2010-01-26 15:13:25 +11001715/*
1716 * Compare function is more complex than it needs to be because
1717 * the return value is only 32 bits and we are doing comparisons
1718 * on 64 bit values
1719 */
1720static int
1721xfs_buf_cmp(
1722 void *priv,
1723 struct list_head *a,
1724 struct list_head *b)
1725{
1726 struct xfs_buf *ap = container_of(a, struct xfs_buf, b_list);
1727 struct xfs_buf *bp = container_of(b, struct xfs_buf, b_list);
1728 xfs_daddr_t diff;
1729
Mark Tinguelyf4b42422012-12-04 17:18:02 -06001730 diff = ap->b_maps[0].bm_bn - bp->b_maps[0].bm_bn;
Dave Chinner089716a2010-01-26 15:13:25 +11001731 if (diff < 0)
1732 return -1;
1733 if (diff > 0)
1734 return 1;
1735 return 0;
1736}
1737
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001738static int
1739__xfs_buf_delwri_submit(
1740 struct list_head *buffer_list,
1741 struct list_head *io_list,
1742 bool wait)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001743{
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001744 struct blk_plug plug;
1745 struct xfs_buf *bp, *n;
1746 int pinned = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001747
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001748 list_for_each_entry_safe(bp, n, buffer_list, b_list) {
1749 if (!wait) {
1750 if (xfs_buf_ispinned(bp)) {
1751 pinned++;
1752 continue;
1753 }
1754 if (!xfs_buf_trylock(bp))
1755 continue;
1756 } else {
1757 xfs_buf_lock(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001758 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001759
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001760 /*
1761 * Someone else might have written the buffer synchronously or
1762 * marked it stale in the meantime. In that case only the
1763 * _XBF_DELWRI_Q flag got cleared, and we have to drop the
1764 * reference and remove it from the list here.
1765 */
1766 if (!(bp->b_flags & _XBF_DELWRI_Q)) {
1767 list_del_init(&bp->b_list);
1768 xfs_buf_relse(bp);
1769 continue;
1770 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001771
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001772 list_move_tail(&bp->b_list, io_list);
1773 trace_xfs_buf_delwri_split(bp, _RET_IP_);
1774 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001775
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001776 list_sort(NULL, io_list, xfs_buf_cmp);
Christoph Hellwiga1b7ea52011-03-30 11:05:09 +00001777
1778 blk_start_plug(&plug);
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001779 list_for_each_entry_safe(bp, n, io_list, b_list) {
1780 bp->b_flags &= ~(_XBF_DELWRI_Q | XBF_ASYNC);
1781 bp->b_flags |= XBF_WRITE;
1782
1783 if (!wait) {
1784 bp->b_flags |= XBF_ASYNC;
1785 list_del_init(&bp->b_list);
Dave Chinner089716a2010-01-26 15:13:25 +11001786 }
Christoph Hellwig939d7232010-07-20 17:51:16 +10001787 xfs_bdstrat_cb(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001788 }
Christoph Hellwiga1b7ea52011-03-30 11:05:09 +00001789 blk_finish_plug(&plug);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001790
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001791 return pinned;
1792}
Nathan Scottf07c2252006-09-28 10:52:15 +10001793
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001794/*
1795 * Write out a buffer list asynchronously.
1796 *
1797 * This will take the @buffer_list, write all non-locked and non-pinned buffers
1798 * out and not wait for I/O completion on any of the buffers. This interface
1799 * is only safely useable for callers that can track I/O completion by higher
1800 * level means, e.g. AIL pushing as the @buffer_list is consumed in this
1801 * function.
1802 */
1803int
1804xfs_buf_delwri_submit_nowait(
1805 struct list_head *buffer_list)
1806{
1807 LIST_HEAD (io_list);
1808 return __xfs_buf_delwri_submit(buffer_list, &io_list, false);
1809}
1810
1811/*
1812 * Write out a buffer list synchronously.
1813 *
1814 * This will take the @buffer_list, write all buffers out and wait for I/O
1815 * completion on all of the buffers. @buffer_list is consumed by the function,
1816 * so callers must have some other way of tracking buffers if they require such
1817 * functionality.
1818 */
1819int
1820xfs_buf_delwri_submit(
1821 struct list_head *buffer_list)
1822{
1823 LIST_HEAD (io_list);
1824 int error = 0, error2;
1825 struct xfs_buf *bp;
1826
1827 __xfs_buf_delwri_submit(buffer_list, &io_list, true);
1828
1829 /* Wait for IO to complete. */
1830 while (!list_empty(&io_list)) {
1831 bp = list_first_entry(&io_list, struct xfs_buf, b_list);
1832
1833 list_del_init(&bp->b_list);
1834 error2 = xfs_buf_iowait(bp);
1835 xfs_buf_relse(bp);
1836 if (!error)
1837 error = error2;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001838 }
1839
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001840 return error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001841}
1842
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001843int __init
Nathan Scottce8e9222006-01-11 15:39:08 +11001844xfs_buf_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001845{
Nathan Scott87582802006-03-14 13:18:19 +11001846 xfs_buf_zone = kmem_zone_init_flags(sizeof(xfs_buf_t), "xfs_buf",
1847 KM_ZONE_HWALIGN, NULL);
Nathan Scottce8e9222006-01-11 15:39:08 +11001848 if (!xfs_buf_zone)
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001849 goto out;
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001850
Dave Chinner51749e42010-09-08 09:00:22 +00001851 xfslogd_workqueue = alloc_workqueue("xfslogd",
Tejun Heo6370a6a2010-10-11 15:12:27 +02001852 WQ_MEM_RECLAIM | WQ_HIGHPRI, 1);
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001853 if (!xfslogd_workqueue)
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001854 goto out_free_buf_zone;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001855
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001856 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001857
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001858 out_free_buf_zone:
Nathan Scottce8e9222006-01-11 15:39:08 +11001859 kmem_zone_destroy(xfs_buf_zone);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001860 out:
Nathan Scott87582802006-03-14 13:18:19 +11001861 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001862}
1863
Linus Torvalds1da177e2005-04-16 15:20:36 -07001864void
Nathan Scottce8e9222006-01-11 15:39:08 +11001865xfs_buf_terminate(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001866{
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001867 destroy_workqueue(xfslogd_workqueue);
Nathan Scottce8e9222006-01-11 15:39:08 +11001868 kmem_zone_destroy(xfs_buf_zone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001869}