blob: 82b70bda9f47a51eb3f31a50ae4fcd32f6c57047 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Nathan Scottf07c2252006-09-28 10:52:15 +10002 * Copyright (c) 2000-2006 Silicon Graphics, Inc.
Nathan Scott7b718762005-11-02 14:58:39 +11003 * All Rights Reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
Nathan Scott7b718762005-11-02 14:58:39 +11005 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License as
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 * published by the Free Software Foundation.
8 *
Nathan Scott7b718762005-11-02 14:58:39 +11009 * This program is distributed in the hope that it would be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 *
Nathan Scott7b718762005-11-02 14:58:39 +110014 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
Linus Torvalds1da177e2005-04-16 15:20:36 -070017 */
Vlad Apostolov93c189c2006-11-11 18:03:49 +110018#include "xfs.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/stddef.h>
20#include <linux/errno.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090021#include <linux/gfp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022#include <linux/pagemap.h>
23#include <linux/init.h>
24#include <linux/vmalloc.h>
25#include <linux/bio.h>
26#include <linux/sysctl.h>
27#include <linux/proc_fs.h>
28#include <linux/workqueue.h>
29#include <linux/percpu.h>
30#include <linux/blkdev.h>
31#include <linux/hash.h>
Christoph Hellwig4df08c52005-09-05 08:34:18 +100032#include <linux/kthread.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080033#include <linux/migrate.h>
Andrew Morton3fcfab12006-10-19 23:28:16 -070034#include <linux/backing-dev.h>
Nigel Cunningham7dfb7102006-12-06 20:34:23 -080035#include <linux/freezer.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070036
Christoph Hellwigb7963132009-03-03 14:48:37 -050037#include "xfs_sb.h"
Dave Chinnered3b4d62010-05-21 12:07:08 +100038#include "xfs_log.h"
Christoph Hellwigb7963132009-03-03 14:48:37 -050039#include "xfs_ag.h"
Christoph Hellwigb7963132009-03-03 14:48:37 -050040#include "xfs_mount.h"
Christoph Hellwig0b1b2132009-12-14 23:14:59 +000041#include "xfs_trace.h"
Christoph Hellwigb7963132009-03-03 14:48:37 -050042
David Chinner7989cb82007-02-10 18:34:56 +110043static kmem_zone_t *xfs_buf_zone;
Christoph Hellwig23ea4032005-06-21 15:14:01 +100044
David Chinner7989cb82007-02-10 18:34:56 +110045static struct workqueue_struct *xfslogd_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070046
Nathan Scottce8e9222006-01-11 15:39:08 +110047#ifdef XFS_BUF_LOCK_TRACKING
48# define XB_SET_OWNER(bp) ((bp)->b_last_holder = current->pid)
49# define XB_CLEAR_OWNER(bp) ((bp)->b_last_holder = -1)
50# define XB_GET_OWNER(bp) ((bp)->b_last_holder)
Linus Torvalds1da177e2005-04-16 15:20:36 -070051#else
Nathan Scottce8e9222006-01-11 15:39:08 +110052# define XB_SET_OWNER(bp) do { } while (0)
53# define XB_CLEAR_OWNER(bp) do { } while (0)
54# define XB_GET_OWNER(bp) do { } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -070055#endif
56
Nathan Scottce8e9222006-01-11 15:39:08 +110057#define xb_to_gfp(flags) \
Dave Chinneraa5c1582012-04-23 15:58:56 +100058 ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : GFP_NOFS) | __GFP_NOWARN)
Linus Torvalds1da177e2005-04-16 15:20:36 -070059
Linus Torvalds1da177e2005-04-16 15:20:36 -070060
James Bottomley73c77e22010-01-25 11:42:24 -060061static inline int
62xfs_buf_is_vmapped(
63 struct xfs_buf *bp)
64{
65 /*
66 * Return true if the buffer is vmapped.
67 *
Dave Chinner611c9942012-04-23 15:59:07 +100068 * b_addr is null if the buffer is not mapped, but the code is clever
69 * enough to know it doesn't have to map a single page, so the check has
70 * to be both for b_addr and bp->b_page_count > 1.
James Bottomley73c77e22010-01-25 11:42:24 -060071 */
Dave Chinner611c9942012-04-23 15:59:07 +100072 return bp->b_addr && bp->b_page_count > 1;
James Bottomley73c77e22010-01-25 11:42:24 -060073}
74
75static inline int
76xfs_buf_vmap_len(
77 struct xfs_buf *bp)
78{
79 return (bp->b_page_count * PAGE_SIZE) - bp->b_offset;
80}
81
Linus Torvalds1da177e2005-04-16 15:20:36 -070082/*
Dave Chinner430cbeb2010-12-02 16:30:55 +110083 * xfs_buf_lru_add - add a buffer to the LRU.
84 *
85 * The LRU takes a new reference to the buffer so that it will only be freed
86 * once the shrinker takes the buffer off the LRU.
Linus Torvalds1da177e2005-04-16 15:20:36 -070087 */
Dave Chinner430cbeb2010-12-02 16:30:55 +110088STATIC void
89xfs_buf_lru_add(
90 struct xfs_buf *bp)
91{
92 struct xfs_buftarg *btp = bp->b_target;
93
94 spin_lock(&btp->bt_lru_lock);
95 if (list_empty(&bp->b_lru)) {
96 atomic_inc(&bp->b_hold);
97 list_add_tail(&bp->b_lru, &btp->bt_lru);
98 btp->bt_lru_nr++;
Carlos Maiolino6fb8a902012-08-10 15:01:51 -030099 bp->b_lru_flags &= ~_XBF_LRU_DISPOSE;
Dave Chinner430cbeb2010-12-02 16:30:55 +1100100 }
101 spin_unlock(&btp->bt_lru_lock);
102}
103
104/*
105 * xfs_buf_lru_del - remove a buffer from the LRU
106 *
107 * The unlocked check is safe here because it only occurs when there are not
108 * b_lru_ref counts left on the inode under the pag->pag_buf_lock. it is there
109 * to optimise the shrinker removing the buffer from the LRU and calling
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300110 * xfs_buf_free(). i.e. it removes an unnecessary round trip on the
Dave Chinner430cbeb2010-12-02 16:30:55 +1100111 * bt_lru_lock.
112 */
113STATIC void
114xfs_buf_lru_del(
115 struct xfs_buf *bp)
116{
117 struct xfs_buftarg *btp = bp->b_target;
118
119 if (list_empty(&bp->b_lru))
120 return;
121
122 spin_lock(&btp->bt_lru_lock);
123 if (!list_empty(&bp->b_lru)) {
124 list_del_init(&bp->b_lru);
125 btp->bt_lru_nr--;
126 }
127 spin_unlock(&btp->bt_lru_lock);
128}
129
130/*
131 * When we mark a buffer stale, we remove the buffer from the LRU and clear the
132 * b_lru_ref count so that the buffer is freed immediately when the buffer
133 * reference count falls to zero. If the buffer is already on the LRU, we need
134 * to remove the reference that LRU holds on the buffer.
135 *
136 * This prevents build-up of stale buffers on the LRU.
137 */
138void
139xfs_buf_stale(
140 struct xfs_buf *bp)
141{
Christoph Hellwig43ff2122012-04-23 15:58:39 +1000142 ASSERT(xfs_buf_islocked(bp));
143
Dave Chinner430cbeb2010-12-02 16:30:55 +1100144 bp->b_flags |= XBF_STALE;
Christoph Hellwig43ff2122012-04-23 15:58:39 +1000145
146 /*
147 * Clear the delwri status so that a delwri queue walker will not
148 * flush this buffer to disk now that it is stale. The delwri queue has
149 * a reference to the buffer, so this is safe to do.
150 */
151 bp->b_flags &= ~_XBF_DELWRI_Q;
152
Dave Chinner430cbeb2010-12-02 16:30:55 +1100153 atomic_set(&(bp)->b_lru_ref, 0);
154 if (!list_empty(&bp->b_lru)) {
155 struct xfs_buftarg *btp = bp->b_target;
156
157 spin_lock(&btp->bt_lru_lock);
Carlos Maiolino6fb8a902012-08-10 15:01:51 -0300158 if (!list_empty(&bp->b_lru) &&
159 !(bp->b_lru_flags & _XBF_LRU_DISPOSE)) {
Dave Chinner430cbeb2010-12-02 16:30:55 +1100160 list_del_init(&bp->b_lru);
161 btp->bt_lru_nr--;
162 atomic_dec(&bp->b_hold);
163 }
164 spin_unlock(&btp->bt_lru_lock);
165 }
166 ASSERT(atomic_read(&bp->b_hold) >= 1);
167}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168
Dave Chinner3e85c862012-06-22 18:50:09 +1000169static int
170xfs_buf_get_maps(
171 struct xfs_buf *bp,
172 int map_count)
173{
174 ASSERT(bp->b_maps == NULL);
175 bp->b_map_count = map_count;
176
177 if (map_count == 1) {
Mark Tinguelyf4b42422012-12-04 17:18:02 -0600178 bp->b_maps = &bp->__b_map;
Dave Chinner3e85c862012-06-22 18:50:09 +1000179 return 0;
180 }
181
182 bp->b_maps = kmem_zalloc(map_count * sizeof(struct xfs_buf_map),
183 KM_NOFS);
184 if (!bp->b_maps)
185 return ENOMEM;
186 return 0;
187}
188
189/*
190 * Frees b_pages if it was allocated.
191 */
192static void
193xfs_buf_free_maps(
194 struct xfs_buf *bp)
195{
Mark Tinguelyf4b42422012-12-04 17:18:02 -0600196 if (bp->b_maps != &bp->__b_map) {
Dave Chinner3e85c862012-06-22 18:50:09 +1000197 kmem_free(bp->b_maps);
198 bp->b_maps = NULL;
199 }
200}
201
Christoph Hellwig4347b9d2011-10-10 16:52:48 +0000202struct xfs_buf *
Dave Chinner3e85c862012-06-22 18:50:09 +1000203_xfs_buf_alloc(
Christoph Hellwig4347b9d2011-10-10 16:52:48 +0000204 struct xfs_buftarg *target,
Dave Chinner3e85c862012-06-22 18:50:09 +1000205 struct xfs_buf_map *map,
206 int nmaps,
Nathan Scottce8e9222006-01-11 15:39:08 +1100207 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208{
Christoph Hellwig4347b9d2011-10-10 16:52:48 +0000209 struct xfs_buf *bp;
Dave Chinner3e85c862012-06-22 18:50:09 +1000210 int error;
211 int i;
Christoph Hellwig4347b9d2011-10-10 16:52:48 +0000212
Dave Chinneraa5c1582012-04-23 15:58:56 +1000213 bp = kmem_zone_zalloc(xfs_buf_zone, KM_NOFS);
Christoph Hellwig4347b9d2011-10-10 16:52:48 +0000214 if (unlikely(!bp))
215 return NULL;
216
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217 /*
Dave Chinner12bcb3f2012-04-23 15:59:05 +1000218 * We don't want certain flags to appear in b_flags unless they are
219 * specifically set by later operations on the buffer.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220 */
Dave Chinner611c9942012-04-23 15:59:07 +1000221 flags &= ~(XBF_UNMAPPED | XBF_TRYLOCK | XBF_ASYNC | XBF_READ_AHEAD);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222
Nathan Scottce8e9222006-01-11 15:39:08 +1100223 atomic_set(&bp->b_hold, 1);
Dave Chinner430cbeb2010-12-02 16:30:55 +1100224 atomic_set(&bp->b_lru_ref, 1);
David Chinnerb4dd3302008-08-13 16:36:11 +1000225 init_completion(&bp->b_iowait);
Dave Chinner430cbeb2010-12-02 16:30:55 +1100226 INIT_LIST_HEAD(&bp->b_lru);
Nathan Scottce8e9222006-01-11 15:39:08 +1100227 INIT_LIST_HEAD(&bp->b_list);
Dave Chinner74f75a02010-09-24 19:59:04 +1000228 RB_CLEAR_NODE(&bp->b_rbnode);
Thomas Gleixnera731cd12010-09-07 14:33:15 +0000229 sema_init(&bp->b_sema, 0); /* held, no waiters */
Nathan Scottce8e9222006-01-11 15:39:08 +1100230 XB_SET_OWNER(bp);
231 bp->b_target = target;
Dave Chinner3e85c862012-06-22 18:50:09 +1000232 bp->b_flags = flags;
Dave Chinnerde1cbee2012-04-23 15:58:50 +1000233
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234 /*
Dave Chinneraa0e8832012-04-23 15:58:52 +1000235 * Set length and io_length to the same value initially.
236 * I/O routines should use io_length, which will be the same in
Linus Torvalds1da177e2005-04-16 15:20:36 -0700237 * most cases but may be reset (e.g. XFS recovery).
238 */
Dave Chinner3e85c862012-06-22 18:50:09 +1000239 error = xfs_buf_get_maps(bp, nmaps);
240 if (error) {
241 kmem_zone_free(xfs_buf_zone, bp);
242 return NULL;
243 }
244
245 bp->b_bn = map[0].bm_bn;
246 bp->b_length = 0;
247 for (i = 0; i < nmaps; i++) {
248 bp->b_maps[i].bm_bn = map[i].bm_bn;
249 bp->b_maps[i].bm_len = map[i].bm_len;
250 bp->b_length += map[i].bm_len;
251 }
252 bp->b_io_length = bp->b_length;
253
Nathan Scottce8e9222006-01-11 15:39:08 +1100254 atomic_set(&bp->b_pin_count, 0);
255 init_waitqueue_head(&bp->b_waiters);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256
Nathan Scottce8e9222006-01-11 15:39:08 +1100257 XFS_STATS_INC(xb_create);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000258 trace_xfs_buf_init(bp, _RET_IP_);
Christoph Hellwig4347b9d2011-10-10 16:52:48 +0000259
260 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700261}
262
263/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100264 * Allocate a page array capable of holding a specified number
265 * of pages, and point the page buf at it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266 */
267STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100268_xfs_buf_get_pages(
269 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270 int page_count,
Nathan Scottce8e9222006-01-11 15:39:08 +1100271 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272{
273 /* Make sure that we have a page list */
Nathan Scottce8e9222006-01-11 15:39:08 +1100274 if (bp->b_pages == NULL) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100275 bp->b_page_count = page_count;
276 if (page_count <= XB_PAGES) {
277 bp->b_pages = bp->b_page_array;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100279 bp->b_pages = kmem_alloc(sizeof(struct page *) *
Dave Chinneraa5c1582012-04-23 15:58:56 +1000280 page_count, KM_NOFS);
Nathan Scottce8e9222006-01-11 15:39:08 +1100281 if (bp->b_pages == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282 return -ENOMEM;
283 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100284 memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700285 }
286 return 0;
287}
288
289/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100290 * Frees b_pages if it was allocated.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291 */
292STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +1100293_xfs_buf_free_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700294 xfs_buf_t *bp)
295{
Nathan Scottce8e9222006-01-11 15:39:08 +1100296 if (bp->b_pages != bp->b_page_array) {
Denys Vlasenkof0e2d932008-05-19 16:31:57 +1000297 kmem_free(bp->b_pages);
Dave Chinner3fc98b12009-12-14 23:11:57 +0000298 bp->b_pages = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299 }
300}
301
302/*
303 * Releases the specified buffer.
304 *
305 * The modification state of any associated pages is left unchanged.
Nathan Scottce8e9222006-01-11 15:39:08 +1100306 * The buffer most not be on any hash - use xfs_buf_rele instead for
Linus Torvalds1da177e2005-04-16 15:20:36 -0700307 * hashed and refcounted buffers
308 */
309void
Nathan Scottce8e9222006-01-11 15:39:08 +1100310xfs_buf_free(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311 xfs_buf_t *bp)
312{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000313 trace_xfs_buf_free(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314
Dave Chinner430cbeb2010-12-02 16:30:55 +1100315 ASSERT(list_empty(&bp->b_lru));
316
Dave Chinner0e6e8472011-03-26 09:16:45 +1100317 if (bp->b_flags & _XBF_PAGES) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318 uint i;
319
James Bottomley73c77e22010-01-25 11:42:24 -0600320 if (xfs_buf_is_vmapped(bp))
Alex Elder8a262e52010-03-16 18:55:56 +0000321 vm_unmap_ram(bp->b_addr - bp->b_offset,
322 bp->b_page_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323
Nathan Scott948ecdb2006-09-28 11:03:13 +1000324 for (i = 0; i < bp->b_page_count; i++) {
325 struct page *page = bp->b_pages[i];
326
Dave Chinner0e6e8472011-03-26 09:16:45 +1100327 __free_page(page);
Nathan Scott948ecdb2006-09-28 11:03:13 +1000328 }
Dave Chinner0e6e8472011-03-26 09:16:45 +1100329 } else if (bp->b_flags & _XBF_KMEM)
330 kmem_free(bp->b_addr);
Dave Chinner3fc98b12009-12-14 23:11:57 +0000331 _xfs_buf_free_pages(bp);
Dave Chinner3e85c862012-06-22 18:50:09 +1000332 xfs_buf_free_maps(bp);
Christoph Hellwig4347b9d2011-10-10 16:52:48 +0000333 kmem_zone_free(xfs_buf_zone, bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700334}
335
336/*
Dave Chinner0e6e8472011-03-26 09:16:45 +1100337 * Allocates all the pages for buffer in question and builds it's page list.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700338 */
339STATIC int
Dave Chinner0e6e8472011-03-26 09:16:45 +1100340xfs_buf_allocate_memory(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700341 xfs_buf_t *bp,
342 uint flags)
343{
Dave Chinneraa0e8832012-04-23 15:58:52 +1000344 size_t size;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700345 size_t nbytes, offset;
Nathan Scottce8e9222006-01-11 15:39:08 +1100346 gfp_t gfp_mask = xb_to_gfp(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347 unsigned short page_count, i;
Dave Chinner795cac72012-04-23 15:58:53 +1000348 xfs_off_t start, end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700349 int error;
350
Dave Chinner0e6e8472011-03-26 09:16:45 +1100351 /*
352 * for buffers that are contained within a single page, just allocate
353 * the memory from the heap - there's no need for the complexity of
354 * page arrays to keep allocation down to order 0.
355 */
Dave Chinner795cac72012-04-23 15:58:53 +1000356 size = BBTOB(bp->b_length);
357 if (size < PAGE_SIZE) {
Dave Chinneraa5c1582012-04-23 15:58:56 +1000358 bp->b_addr = kmem_alloc(size, KM_NOFS);
Dave Chinner0e6e8472011-03-26 09:16:45 +1100359 if (!bp->b_addr) {
360 /* low memory - use alloc_page loop instead */
361 goto use_alloc_page;
362 }
363
Dave Chinner795cac72012-04-23 15:58:53 +1000364 if (((unsigned long)(bp->b_addr + size - 1) & PAGE_MASK) !=
Dave Chinner0e6e8472011-03-26 09:16:45 +1100365 ((unsigned long)bp->b_addr & PAGE_MASK)) {
366 /* b_addr spans two pages - use alloc_page instead */
367 kmem_free(bp->b_addr);
368 bp->b_addr = NULL;
369 goto use_alloc_page;
370 }
371 bp->b_offset = offset_in_page(bp->b_addr);
372 bp->b_pages = bp->b_page_array;
373 bp->b_pages[0] = virt_to_page(bp->b_addr);
374 bp->b_page_count = 1;
Dave Chinner611c9942012-04-23 15:59:07 +1000375 bp->b_flags |= _XBF_KMEM;
Dave Chinner0e6e8472011-03-26 09:16:45 +1100376 return 0;
377 }
378
379use_alloc_page:
Mark Tinguelyf4b42422012-12-04 17:18:02 -0600380 start = BBTOB(bp->b_maps[0].bm_bn) >> PAGE_SHIFT;
381 end = (BBTOB(bp->b_maps[0].bm_bn + bp->b_length) + PAGE_SIZE - 1)
Dave Chinnercbb7baab2012-06-22 18:50:08 +1000382 >> PAGE_SHIFT;
Dave Chinner795cac72012-04-23 15:58:53 +1000383 page_count = end - start;
Nathan Scottce8e9222006-01-11 15:39:08 +1100384 error = _xfs_buf_get_pages(bp, page_count, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385 if (unlikely(error))
386 return error;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700387
Nathan Scottce8e9222006-01-11 15:39:08 +1100388 offset = bp->b_offset;
Dave Chinner0e6e8472011-03-26 09:16:45 +1100389 bp->b_flags |= _XBF_PAGES;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390
Nathan Scottce8e9222006-01-11 15:39:08 +1100391 for (i = 0; i < bp->b_page_count; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700392 struct page *page;
393 uint retries = 0;
Dave Chinner0e6e8472011-03-26 09:16:45 +1100394retry:
395 page = alloc_page(gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396 if (unlikely(page == NULL)) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100397 if (flags & XBF_READ_AHEAD) {
398 bp->b_page_count = i;
Dave Chinner0e6e8472011-03-26 09:16:45 +1100399 error = ENOMEM;
400 goto out_free_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401 }
402
403 /*
404 * This could deadlock.
405 *
406 * But until all the XFS lowlevel code is revamped to
407 * handle buffer allocation failures we can't do much.
408 */
409 if (!(++retries % 100))
Dave Chinner4f107002011-03-07 10:00:35 +1100410 xfs_err(NULL,
411 "possible memory allocation deadlock in %s (mode:0x%x)",
Harvey Harrison34a622b2008-04-10 12:19:21 +1000412 __func__, gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413
Nathan Scottce8e9222006-01-11 15:39:08 +1100414 XFS_STATS_INC(xb_page_retries);
Jens Axboe8aa7e842009-07-09 14:52:32 +0200415 congestion_wait(BLK_RW_ASYNC, HZ/50);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416 goto retry;
417 }
418
Nathan Scottce8e9222006-01-11 15:39:08 +1100419 XFS_STATS_INC(xb_page_found);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420
Dave Chinner0e6e8472011-03-26 09:16:45 +1100421 nbytes = min_t(size_t, size, PAGE_SIZE - offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422 size -= nbytes;
Nathan Scottce8e9222006-01-11 15:39:08 +1100423 bp->b_pages[i] = page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424 offset = 0;
425 }
Dave Chinner0e6e8472011-03-26 09:16:45 +1100426 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427
Dave Chinner0e6e8472011-03-26 09:16:45 +1100428out_free_pages:
429 for (i = 0; i < bp->b_page_count; i++)
430 __free_page(bp->b_pages[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700431 return error;
432}
433
434/*
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300435 * Map buffer into kernel address-space if necessary.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436 */
437STATIC int
Nathan Scottce8e9222006-01-11 15:39:08 +1100438_xfs_buf_map_pages(
Linus Torvalds1da177e2005-04-16 15:20:36 -0700439 xfs_buf_t *bp,
440 uint flags)
441{
Dave Chinner0e6e8472011-03-26 09:16:45 +1100442 ASSERT(bp->b_flags & _XBF_PAGES);
Nathan Scottce8e9222006-01-11 15:39:08 +1100443 if (bp->b_page_count == 1) {
Dave Chinner0e6e8472011-03-26 09:16:45 +1100444 /* A single page buffer is always mappable */
Nathan Scottce8e9222006-01-11 15:39:08 +1100445 bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
Dave Chinner611c9942012-04-23 15:59:07 +1000446 } else if (flags & XBF_UNMAPPED) {
447 bp->b_addr = NULL;
448 } else {
Dave Chinnera19fb382011-03-26 09:13:42 +1100449 int retried = 0;
450
451 do {
452 bp->b_addr = vm_map_ram(bp->b_pages, bp->b_page_count,
453 -1, PAGE_KERNEL);
454 if (bp->b_addr)
455 break;
456 vm_unmap_aliases();
457 } while (retried++ <= 1);
458
459 if (!bp->b_addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700460 return -ENOMEM;
Nathan Scottce8e9222006-01-11 15:39:08 +1100461 bp->b_addr += bp->b_offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700462 }
463
464 return 0;
465}
466
467/*
468 * Finding and Reading Buffers
469 */
470
471/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100472 * Look up, and creates if absent, a lockable buffer for
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473 * a given range of an inode. The buffer is returned
Chandra Seetharamaneabbaf12011-09-08 20:18:50 +0000474 * locked. No I/O is implied by this call.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700475 */
476xfs_buf_t *
Nathan Scottce8e9222006-01-11 15:39:08 +1100477_xfs_buf_find(
Dave Chinnere70b73f2012-04-23 15:58:49 +1000478 struct xfs_buftarg *btp,
Dave Chinner3e85c862012-06-22 18:50:09 +1000479 struct xfs_buf_map *map,
480 int nmaps,
Nathan Scottce8e9222006-01-11 15:39:08 +1100481 xfs_buf_flags_t flags,
482 xfs_buf_t *new_bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483{
Dave Chinnere70b73f2012-04-23 15:58:49 +1000484 size_t numbytes;
Dave Chinner74f75a02010-09-24 19:59:04 +1000485 struct xfs_perag *pag;
486 struct rb_node **rbp;
487 struct rb_node *parent;
488 xfs_buf_t *bp;
Dave Chinner3e85c862012-06-22 18:50:09 +1000489 xfs_daddr_t blkno = map[0].bm_bn;
Dave Chinner10616b82013-01-21 23:53:52 +1100490 xfs_daddr_t eofs;
Dave Chinner3e85c862012-06-22 18:50:09 +1000491 int numblks = 0;
492 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700493
Dave Chinner3e85c862012-06-22 18:50:09 +1000494 for (i = 0; i < nmaps; i++)
495 numblks += map[i].bm_len;
Dave Chinnere70b73f2012-04-23 15:58:49 +1000496 numbytes = BBTOB(numblks);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497
498 /* Check for IOs smaller than the sector size / not sector aligned */
Dave Chinnere70b73f2012-04-23 15:58:49 +1000499 ASSERT(!(numbytes < (1 << btp->bt_sshift)));
Dave Chinnerde1cbee2012-04-23 15:58:50 +1000500 ASSERT(!(BBTOB(blkno) & (xfs_off_t)btp->bt_smask));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700501
Dave Chinner10616b82013-01-21 23:53:52 +1100502 /*
503 * Corrupted block numbers can get through to here, unfortunately, so we
504 * have to check that the buffer falls within the filesystem bounds.
505 */
506 eofs = XFS_FSB_TO_BB(btp->bt_mount, btp->bt_mount->m_sb.sb_dblocks);
507 if (blkno >= eofs) {
508 /*
509 * XXX (dgc): we should really be returning EFSCORRUPTED here,
510 * but none of the higher level infrastructure supports
511 * returning a specific error on buffer lookup failures.
512 */
513 xfs_alert(btp->bt_mount,
514 "%s: Block out of range: block 0x%llx, EOFS 0x%llx ",
515 __func__, blkno, eofs);
516 return NULL;
517 }
518
Dave Chinner74f75a02010-09-24 19:59:04 +1000519 /* get tree root */
520 pag = xfs_perag_get(btp->bt_mount,
Dave Chinnere70b73f2012-04-23 15:58:49 +1000521 xfs_daddr_to_agno(btp->bt_mount, blkno));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700522
Dave Chinner74f75a02010-09-24 19:59:04 +1000523 /* walk tree */
524 spin_lock(&pag->pag_buf_lock);
525 rbp = &pag->pag_buf_tree.rb_node;
526 parent = NULL;
527 bp = NULL;
528 while (*rbp) {
529 parent = *rbp;
530 bp = rb_entry(parent, struct xfs_buf, b_rbnode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700531
Dave Chinnerde1cbee2012-04-23 15:58:50 +1000532 if (blkno < bp->b_bn)
Dave Chinner74f75a02010-09-24 19:59:04 +1000533 rbp = &(*rbp)->rb_left;
Dave Chinnerde1cbee2012-04-23 15:58:50 +1000534 else if (blkno > bp->b_bn)
Dave Chinner74f75a02010-09-24 19:59:04 +1000535 rbp = &(*rbp)->rb_right;
536 else {
537 /*
Dave Chinnerde1cbee2012-04-23 15:58:50 +1000538 * found a block number match. If the range doesn't
Dave Chinner74f75a02010-09-24 19:59:04 +1000539 * match, the only way this is allowed is if the buffer
540 * in the cache is stale and the transaction that made
541 * it stale has not yet committed. i.e. we are
542 * reallocating a busy extent. Skip this buffer and
543 * continue searching to the right for an exact match.
544 */
Dave Chinner4e94b712012-04-23 15:58:51 +1000545 if (bp->b_length != numblks) {
Dave Chinner74f75a02010-09-24 19:59:04 +1000546 ASSERT(bp->b_flags & XBF_STALE);
547 rbp = &(*rbp)->rb_right;
548 continue;
549 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100550 atomic_inc(&bp->b_hold);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700551 goto found;
552 }
553 }
554
555 /* No match found */
Nathan Scottce8e9222006-01-11 15:39:08 +1100556 if (new_bp) {
Dave Chinner74f75a02010-09-24 19:59:04 +1000557 rb_link_node(&new_bp->b_rbnode, parent, rbp);
558 rb_insert_color(&new_bp->b_rbnode, &pag->pag_buf_tree);
559 /* the buffer keeps the perag reference until it is freed */
560 new_bp->b_pag = pag;
561 spin_unlock(&pag->pag_buf_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700562 } else {
Nathan Scottce8e9222006-01-11 15:39:08 +1100563 XFS_STATS_INC(xb_miss_locked);
Dave Chinner74f75a02010-09-24 19:59:04 +1000564 spin_unlock(&pag->pag_buf_lock);
565 xfs_perag_put(pag);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700566 }
Nathan Scottce8e9222006-01-11 15:39:08 +1100567 return new_bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700568
569found:
Dave Chinner74f75a02010-09-24 19:59:04 +1000570 spin_unlock(&pag->pag_buf_lock);
571 xfs_perag_put(pag);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572
Christoph Hellwig0c842ad2011-07-08 14:36:19 +0200573 if (!xfs_buf_trylock(bp)) {
574 if (flags & XBF_TRYLOCK) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100575 xfs_buf_rele(bp);
576 XFS_STATS_INC(xb_busy_locked);
577 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578 }
Christoph Hellwig0c842ad2011-07-08 14:36:19 +0200579 xfs_buf_lock(bp);
580 XFS_STATS_INC(xb_get_locked_waited);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700581 }
582
Dave Chinner0e6e8472011-03-26 09:16:45 +1100583 /*
584 * if the buffer is stale, clear all the external state associated with
585 * it. We need to keep flags such as how we allocated the buffer memory
586 * intact here.
587 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100588 if (bp->b_flags & XBF_STALE) {
589 ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
Dave Chinnercfb02852012-11-12 22:54:19 +1100590 ASSERT(bp->b_iodone == NULL);
Dave Chinner611c9942012-04-23 15:59:07 +1000591 bp->b_flags &= _XBF_KMEM | _XBF_PAGES;
Dave Chinner1813dd62012-11-14 17:54:40 +1100592 bp->b_ops = NULL;
David Chinner2f926582005-09-05 08:33:35 +1000593 }
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000594
595 trace_xfs_buf_find(bp, flags, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +1100596 XFS_STATS_INC(xb_get_locked);
597 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598}
599
600/*
Dave Chinner38158322011-09-30 04:45:02 +0000601 * Assembles a buffer covering the specified range. The code is optimised for
602 * cache hits, as metadata intensive workloads will see 3 orders of magnitude
603 * more hits than misses.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700604 */
Dave Chinner38158322011-09-30 04:45:02 +0000605struct xfs_buf *
Dave Chinner6dde2702012-06-22 18:50:10 +1000606xfs_buf_get_map(
607 struct xfs_buftarg *target,
608 struct xfs_buf_map *map,
609 int nmaps,
Nathan Scottce8e9222006-01-11 15:39:08 +1100610 xfs_buf_flags_t flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611{
Dave Chinner38158322011-09-30 04:45:02 +0000612 struct xfs_buf *bp;
613 struct xfs_buf *new_bp;
Dave Chinner0e6e8472011-03-26 09:16:45 +1100614 int error = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615
Dave Chinner6dde2702012-06-22 18:50:10 +1000616 bp = _xfs_buf_find(target, map, nmaps, flags, NULL);
Dave Chinner38158322011-09-30 04:45:02 +0000617 if (likely(bp))
618 goto found;
619
Dave Chinner6dde2702012-06-22 18:50:10 +1000620 new_bp = _xfs_buf_alloc(target, map, nmaps, flags);
Nathan Scottce8e9222006-01-11 15:39:08 +1100621 if (unlikely(!new_bp))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622 return NULL;
623
Dave Chinnerfe2429b2012-04-23 15:58:45 +1000624 error = xfs_buf_allocate_memory(new_bp, flags);
625 if (error) {
Dave Chinner3e85c862012-06-22 18:50:09 +1000626 xfs_buf_free(new_bp);
Dave Chinner38158322011-09-30 04:45:02 +0000627 return NULL;
628 }
629
Dave Chinner6dde2702012-06-22 18:50:10 +1000630 bp = _xfs_buf_find(target, map, nmaps, flags, new_bp);
Dave Chinnerfe2429b2012-04-23 15:58:45 +1000631 if (!bp) {
632 xfs_buf_free(new_bp);
633 return NULL;
634 }
635
636 if (bp != new_bp)
637 xfs_buf_free(new_bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638
Dave Chinner38158322011-09-30 04:45:02 +0000639found:
Dave Chinner611c9942012-04-23 15:59:07 +1000640 if (!bp->b_addr) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100641 error = _xfs_buf_map_pages(bp, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700642 if (unlikely(error)) {
Dave Chinner4f107002011-03-07 10:00:35 +1100643 xfs_warn(target->bt_mount,
644 "%s: failed to map pages\n", __func__);
Dave Chinnera8acad72012-04-23 15:58:54 +1000645 xfs_buf_relse(bp);
646 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647 }
648 }
649
Nathan Scottce8e9222006-01-11 15:39:08 +1100650 XFS_STATS_INC(xb_get);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000651 trace_xfs_buf_get(bp, flags, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +1100652 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700653}
654
Christoph Hellwig5d765b92008-12-03 12:20:26 +0100655STATIC int
656_xfs_buf_read(
657 xfs_buf_t *bp,
658 xfs_buf_flags_t flags)
659{
Christoph Hellwig43ff2122012-04-23 15:58:39 +1000660 ASSERT(!(flags & XBF_WRITE));
Mark Tinguelyf4b42422012-12-04 17:18:02 -0600661 ASSERT(bp->b_maps[0].bm_bn != XFS_BUF_DADDR_NULL);
Christoph Hellwig5d765b92008-12-03 12:20:26 +0100662
Christoph Hellwig43ff2122012-04-23 15:58:39 +1000663 bp->b_flags &= ~(XBF_WRITE | XBF_ASYNC | XBF_READ_AHEAD);
Christoph Hellwig1d5ae5d2011-07-08 14:36:32 +0200664 bp->b_flags |= flags & (XBF_READ | XBF_ASYNC | XBF_READ_AHEAD);
Christoph Hellwig5d765b92008-12-03 12:20:26 +0100665
Dave Chinner0e95f192012-04-23 15:58:46 +1000666 xfs_buf_iorequest(bp);
667 if (flags & XBF_ASYNC)
668 return 0;
Dave Chinnerec53d1d2010-07-20 17:52:59 +1000669 return xfs_buf_iowait(bp);
Christoph Hellwig5d765b92008-12-03 12:20:26 +0100670}
671
Linus Torvalds1da177e2005-04-16 15:20:36 -0700672xfs_buf_t *
Dave Chinner6dde2702012-06-22 18:50:10 +1000673xfs_buf_read_map(
674 struct xfs_buftarg *target,
675 struct xfs_buf_map *map,
676 int nmaps,
Dave Chinnerc3f8fc72012-11-12 22:54:01 +1100677 xfs_buf_flags_t flags,
Dave Chinner1813dd62012-11-14 17:54:40 +1100678 const struct xfs_buf_ops *ops)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700679{
Dave Chinner6dde2702012-06-22 18:50:10 +1000680 struct xfs_buf *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700681
Nathan Scottce8e9222006-01-11 15:39:08 +1100682 flags |= XBF_READ;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683
Dave Chinner6dde2702012-06-22 18:50:10 +1000684 bp = xfs_buf_get_map(target, map, nmaps, flags);
Nathan Scottce8e9222006-01-11 15:39:08 +1100685 if (bp) {
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000686 trace_xfs_buf_read(bp, flags, _RET_IP_);
687
Nathan Scottce8e9222006-01-11 15:39:08 +1100688 if (!XFS_BUF_ISDONE(bp)) {
Nathan Scottce8e9222006-01-11 15:39:08 +1100689 XFS_STATS_INC(xb_get_read);
Dave Chinner1813dd62012-11-14 17:54:40 +1100690 bp->b_ops = ops;
Christoph Hellwig5d765b92008-12-03 12:20:26 +0100691 _xfs_buf_read(bp, flags);
Nathan Scottce8e9222006-01-11 15:39:08 +1100692 } else if (flags & XBF_ASYNC) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700693 /*
694 * Read ahead call which is already satisfied,
695 * drop the buffer
696 */
Dave Chinnera8acad72012-04-23 15:58:54 +1000697 xfs_buf_relse(bp);
698 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700699 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700700 /* We do not want read in the flags */
Nathan Scottce8e9222006-01-11 15:39:08 +1100701 bp->b_flags &= ~XBF_READ;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700702 }
703 }
704
Nathan Scottce8e9222006-01-11 15:39:08 +1100705 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706}
707
708/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100709 * If we are not low on memory then do the readahead in a deadlock
710 * safe manner.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700711 */
712void
Dave Chinner6dde2702012-06-22 18:50:10 +1000713xfs_buf_readahead_map(
714 struct xfs_buftarg *target,
715 struct xfs_buf_map *map,
Dave Chinnerc3f8fc72012-11-12 22:54:01 +1100716 int nmaps,
Dave Chinner1813dd62012-11-14 17:54:40 +1100717 const struct xfs_buf_ops *ops)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700718{
Dave Chinner0e6e8472011-03-26 09:16:45 +1100719 if (bdi_read_congested(target->bt_bdi))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700720 return;
721
Dave Chinner6dde2702012-06-22 18:50:10 +1000722 xfs_buf_read_map(target, map, nmaps,
Dave Chinner1813dd62012-11-14 17:54:40 +1100723 XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD, ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700724}
725
Dave Chinner5adc94c2010-09-24 21:58:31 +1000726/*
727 * Read an uncached buffer from disk. Allocates and returns a locked
728 * buffer containing the disk contents or nothing.
729 */
730struct xfs_buf *
731xfs_buf_read_uncached(
Dave Chinner5adc94c2010-09-24 21:58:31 +1000732 struct xfs_buftarg *target,
733 xfs_daddr_t daddr,
Dave Chinnere70b73f2012-04-23 15:58:49 +1000734 size_t numblks,
Dave Chinnerc3f8fc72012-11-12 22:54:01 +1100735 int flags,
Dave Chinner1813dd62012-11-14 17:54:40 +1100736 const struct xfs_buf_ops *ops)
Dave Chinner5adc94c2010-09-24 21:58:31 +1000737{
Dave Chinnereab4e632012-11-12 22:54:02 +1100738 struct xfs_buf *bp;
Dave Chinner5adc94c2010-09-24 21:58:31 +1000739
Dave Chinnere70b73f2012-04-23 15:58:49 +1000740 bp = xfs_buf_get_uncached(target, numblks, flags);
Dave Chinner5adc94c2010-09-24 21:58:31 +1000741 if (!bp)
742 return NULL;
743
744 /* set up the buffer for a read IO */
Dave Chinner3e85c862012-06-22 18:50:09 +1000745 ASSERT(bp->b_map_count == 1);
746 bp->b_bn = daddr;
747 bp->b_maps[0].bm_bn = daddr;
Dave Chinnercbb7baab2012-06-22 18:50:08 +1000748 bp->b_flags |= XBF_READ;
Dave Chinner1813dd62012-11-14 17:54:40 +1100749 bp->b_ops = ops;
Dave Chinner5adc94c2010-09-24 21:58:31 +1000750
Dave Chinnere70b73f2012-04-23 15:58:49 +1000751 xfsbdstrat(target->bt_mount, bp);
Dave Chinnereab4e632012-11-12 22:54:02 +1100752 xfs_buf_iowait(bp);
Dave Chinner5adc94c2010-09-24 21:58:31 +1000753 return bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700754}
755
Dave Chinner44396472011-04-21 09:34:27 +0000756/*
757 * Return a buffer allocated as an empty buffer and associated to external
758 * memory via xfs_buf_associate_memory() back to it's empty state.
759 */
760void
761xfs_buf_set_empty(
762 struct xfs_buf *bp,
Dave Chinnere70b73f2012-04-23 15:58:49 +1000763 size_t numblks)
Dave Chinner44396472011-04-21 09:34:27 +0000764{
765 if (bp->b_pages)
766 _xfs_buf_free_pages(bp);
767
768 bp->b_pages = NULL;
769 bp->b_page_count = 0;
770 bp->b_addr = NULL;
Dave Chinner4e94b712012-04-23 15:58:51 +1000771 bp->b_length = numblks;
Dave Chinneraa0e8832012-04-23 15:58:52 +1000772 bp->b_io_length = numblks;
Dave Chinner3e85c862012-06-22 18:50:09 +1000773
774 ASSERT(bp->b_map_count == 1);
Dave Chinner44396472011-04-21 09:34:27 +0000775 bp->b_bn = XFS_BUF_DADDR_NULL;
Dave Chinner3e85c862012-06-22 18:50:09 +1000776 bp->b_maps[0].bm_bn = XFS_BUF_DADDR_NULL;
777 bp->b_maps[0].bm_len = bp->b_length;
Dave Chinner44396472011-04-21 09:34:27 +0000778}
779
Linus Torvalds1da177e2005-04-16 15:20:36 -0700780static inline struct page *
781mem_to_page(
782 void *addr)
783{
Christoph Lameter9e2779f2008-02-04 22:28:34 -0800784 if ((!is_vmalloc_addr(addr))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700785 return virt_to_page(addr);
786 } else {
787 return vmalloc_to_page(addr);
788 }
789}
790
791int
Nathan Scottce8e9222006-01-11 15:39:08 +1100792xfs_buf_associate_memory(
793 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700794 void *mem,
795 size_t len)
796{
797 int rval;
798 int i = 0;
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100799 unsigned long pageaddr;
800 unsigned long offset;
801 size_t buflen;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802 int page_count;
803
Dave Chinner0e6e8472011-03-26 09:16:45 +1100804 pageaddr = (unsigned long)mem & PAGE_MASK;
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100805 offset = (unsigned long)mem - pageaddr;
Dave Chinner0e6e8472011-03-26 09:16:45 +1100806 buflen = PAGE_ALIGN(len + offset);
807 page_count = buflen >> PAGE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700808
809 /* Free any previous set of page pointers */
Nathan Scottce8e9222006-01-11 15:39:08 +1100810 if (bp->b_pages)
811 _xfs_buf_free_pages(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812
Nathan Scottce8e9222006-01-11 15:39:08 +1100813 bp->b_pages = NULL;
814 bp->b_addr = mem;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815
Dave Chinneraa5c1582012-04-23 15:58:56 +1000816 rval = _xfs_buf_get_pages(bp, page_count, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700817 if (rval)
818 return rval;
819
Nathan Scottce8e9222006-01-11 15:39:08 +1100820 bp->b_offset = offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821
Lachlan McIlroyd1afb672007-11-27 17:01:24 +1100822 for (i = 0; i < bp->b_page_count; i++) {
823 bp->b_pages[i] = mem_to_page((void *)pageaddr);
Dave Chinner0e6e8472011-03-26 09:16:45 +1100824 pageaddr += PAGE_SIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700825 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700826
Dave Chinneraa0e8832012-04-23 15:58:52 +1000827 bp->b_io_length = BTOBB(len);
Dave Chinner4e94b712012-04-23 15:58:51 +1000828 bp->b_length = BTOBB(buflen);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829
830 return 0;
831}
832
833xfs_buf_t *
Dave Chinner686865f2010-09-24 20:07:47 +1000834xfs_buf_get_uncached(
835 struct xfs_buftarg *target,
Dave Chinnere70b73f2012-04-23 15:58:49 +1000836 size_t numblks,
Dave Chinner686865f2010-09-24 20:07:47 +1000837 int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700838{
Dave Chinnere70b73f2012-04-23 15:58:49 +1000839 unsigned long page_count;
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000840 int error, i;
Dave Chinner3e85c862012-06-22 18:50:09 +1000841 struct xfs_buf *bp;
842 DEFINE_SINGLE_BUF_MAP(map, XFS_BUF_DADDR_NULL, numblks);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700843
Dave Chinner3e85c862012-06-22 18:50:09 +1000844 bp = _xfs_buf_alloc(target, &map, 1, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700845 if (unlikely(bp == NULL))
846 goto fail;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700847
Dave Chinnere70b73f2012-04-23 15:58:49 +1000848 page_count = PAGE_ALIGN(numblks << BBSHIFT) >> PAGE_SHIFT;
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000849 error = _xfs_buf_get_pages(bp, page_count, 0);
850 if (error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700851 goto fail_free_buf;
852
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000853 for (i = 0; i < page_count; i++) {
Dave Chinner686865f2010-09-24 20:07:47 +1000854 bp->b_pages[i] = alloc_page(xb_to_gfp(flags));
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000855 if (!bp->b_pages[i])
856 goto fail_free_mem;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700857 }
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000858 bp->b_flags |= _XBF_PAGES;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700859
Dave Chinner611c9942012-04-23 15:59:07 +1000860 error = _xfs_buf_map_pages(bp, 0);
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000861 if (unlikely(error)) {
Dave Chinner4f107002011-03-07 10:00:35 +1100862 xfs_warn(target->bt_mount,
863 "%s: failed to map pages\n", __func__);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700864 goto fail_free_mem;
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000865 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700866
Dave Chinner686865f2010-09-24 20:07:47 +1000867 trace_xfs_buf_get_uncached(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700868 return bp;
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000869
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870 fail_free_mem:
Christoph Hellwig1fa40b02007-05-14 18:23:50 +1000871 while (--i >= 0)
872 __free_page(bp->b_pages[i]);
Christoph Hellwigca165b82007-05-24 15:21:11 +1000873 _xfs_buf_free_pages(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700874 fail_free_buf:
Dave Chinner3e85c862012-06-22 18:50:09 +1000875 xfs_buf_free_maps(bp);
Christoph Hellwig4347b9d2011-10-10 16:52:48 +0000876 kmem_zone_free(xfs_buf_zone, bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700877 fail:
878 return NULL;
879}
880
881/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700882 * Increment reference count on buffer, to hold the buffer concurrently
883 * with another thread which may release (free) the buffer asynchronously.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700884 * Must hold the buffer already to call this function.
885 */
886void
Nathan Scottce8e9222006-01-11 15:39:08 +1100887xfs_buf_hold(
888 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000890 trace_xfs_buf_hold(bp, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +1100891 atomic_inc(&bp->b_hold);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700892}
893
894/*
Nathan Scottce8e9222006-01-11 15:39:08 +1100895 * Releases a hold on the specified buffer. If the
896 * the hold count is 1, calls xfs_buf_free.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700897 */
898void
Nathan Scottce8e9222006-01-11 15:39:08 +1100899xfs_buf_rele(
900 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901{
Dave Chinner74f75a02010-09-24 19:59:04 +1000902 struct xfs_perag *pag = bp->b_pag;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000904 trace_xfs_buf_rele(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700905
Dave Chinner74f75a02010-09-24 19:59:04 +1000906 if (!pag) {
Dave Chinner430cbeb2010-12-02 16:30:55 +1100907 ASSERT(list_empty(&bp->b_lru));
Dave Chinner74f75a02010-09-24 19:59:04 +1000908 ASSERT(RB_EMPTY_NODE(&bp->b_rbnode));
Nathan Scottfad3aa12006-02-01 12:14:52 +1100909 if (atomic_dec_and_test(&bp->b_hold))
910 xfs_buf_free(bp);
911 return;
912 }
913
Dave Chinner74f75a02010-09-24 19:59:04 +1000914 ASSERT(!RB_EMPTY_NODE(&bp->b_rbnode));
Dave Chinner430cbeb2010-12-02 16:30:55 +1100915
Lachlan McIlroy37906892008-08-13 15:42:10 +1000916 ASSERT(atomic_read(&bp->b_hold) > 0);
Dave Chinner74f75a02010-09-24 19:59:04 +1000917 if (atomic_dec_and_lock(&bp->b_hold, &pag->pag_buf_lock)) {
Christoph Hellwigbfc60172011-01-07 13:02:23 +0000918 if (!(bp->b_flags & XBF_STALE) &&
Dave Chinner430cbeb2010-12-02 16:30:55 +1100919 atomic_read(&bp->b_lru_ref)) {
920 xfs_buf_lru_add(bp);
921 spin_unlock(&pag->pag_buf_lock);
Christoph Hellwig7f14d0a2005-11-02 15:09:35 +1100922 } else {
Dave Chinner430cbeb2010-12-02 16:30:55 +1100923 xfs_buf_lru_del(bp);
Christoph Hellwig43ff2122012-04-23 15:58:39 +1000924 ASSERT(!(bp->b_flags & _XBF_DELWRI_Q));
Dave Chinner74f75a02010-09-24 19:59:04 +1000925 rb_erase(&bp->b_rbnode, &pag->pag_buf_tree);
926 spin_unlock(&pag->pag_buf_lock);
927 xfs_perag_put(pag);
Nathan Scottce8e9222006-01-11 15:39:08 +1100928 xfs_buf_free(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929 }
930 }
931}
932
933
934/*
Dave Chinner0e6e8472011-03-26 09:16:45 +1100935 * Lock a buffer object, if it is not already locked.
Dave Chinner90810b92010-11-30 15:16:16 +1100936 *
937 * If we come across a stale, pinned, locked buffer, we know that we are
938 * being asked to lock a buffer that has been reallocated. Because it is
939 * pinned, we know that the log has not been pushed to disk and hence it
940 * will still be locked. Rather than continuing to have trylock attempts
941 * fail until someone else pushes the log, push it ourselves before
942 * returning. This means that the xfsaild will not get stuck trying
943 * to push on stale inode buffers.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700944 */
945int
Christoph Hellwig0c842ad2011-07-08 14:36:19 +0200946xfs_buf_trylock(
947 struct xfs_buf *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700948{
949 int locked;
950
Nathan Scottce8e9222006-01-11 15:39:08 +1100951 locked = down_trylock(&bp->b_sema) == 0;
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000952 if (locked)
Nathan Scottce8e9222006-01-11 15:39:08 +1100953 XB_SET_OWNER(bp);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000954
Christoph Hellwig0c842ad2011-07-08 14:36:19 +0200955 trace_xfs_buf_trylock(bp, _RET_IP_);
956 return locked;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700957}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958
959/*
Dave Chinner0e6e8472011-03-26 09:16:45 +1100960 * Lock a buffer object.
Dave Chinnered3b4d62010-05-21 12:07:08 +1000961 *
962 * If we come across a stale, pinned, locked buffer, we know that we
963 * are being asked to lock a buffer that has been reallocated. Because
964 * it is pinned, we know that the log has not been pushed to disk and
965 * hence it will still be locked. Rather than sleeping until someone
966 * else pushes the log, push it ourselves before trying to get the lock.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700967 */
Nathan Scottce8e9222006-01-11 15:39:08 +1100968void
969xfs_buf_lock(
Christoph Hellwig0c842ad2011-07-08 14:36:19 +0200970 struct xfs_buf *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700971{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000972 trace_xfs_buf_lock(bp, _RET_IP_);
973
Dave Chinnered3b4d62010-05-21 12:07:08 +1000974 if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE))
Dave Chinnerebad8612010-09-22 10:47:20 +1000975 xfs_log_force(bp->b_target->bt_mount, 0);
Nathan Scottce8e9222006-01-11 15:39:08 +1100976 down(&bp->b_sema);
977 XB_SET_OWNER(bp);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000978
979 trace_xfs_buf_lock_done(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700980}
981
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982void
Nathan Scottce8e9222006-01-11 15:39:08 +1100983xfs_buf_unlock(
Christoph Hellwig0c842ad2011-07-08 14:36:19 +0200984 struct xfs_buf *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700985{
Nathan Scottce8e9222006-01-11 15:39:08 +1100986 XB_CLEAR_OWNER(bp);
987 up(&bp->b_sema);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000988
989 trace_xfs_buf_unlock(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700990}
991
Nathan Scottce8e9222006-01-11 15:39:08 +1100992STATIC void
993xfs_buf_wait_unpin(
994 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700995{
996 DECLARE_WAITQUEUE (wait, current);
997
Nathan Scottce8e9222006-01-11 15:39:08 +1100998 if (atomic_read(&bp->b_pin_count) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700999 return;
1000
Nathan Scottce8e9222006-01-11 15:39:08 +11001001 add_wait_queue(&bp->b_waiters, &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001002 for (;;) {
1003 set_current_state(TASK_UNINTERRUPTIBLE);
Nathan Scottce8e9222006-01-11 15:39:08 +11001004 if (atomic_read(&bp->b_pin_count) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001005 break;
Jens Axboe7eaceac2011-03-10 08:52:07 +01001006 io_schedule();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001007 }
Nathan Scottce8e9222006-01-11 15:39:08 +11001008 remove_wait_queue(&bp->b_waiters, &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001009 set_current_state(TASK_RUNNING);
1010}
1011
1012/*
1013 * Buffer Utility Routines
1014 */
1015
Linus Torvalds1da177e2005-04-16 15:20:36 -07001016STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001017xfs_buf_iodone_work(
David Howellsc4028952006-11-22 14:57:56 +00001018 struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001019{
Dave Chinner1813dd62012-11-14 17:54:40 +11001020 struct xfs_buf *bp =
David Howellsc4028952006-11-22 14:57:56 +00001021 container_of(work, xfs_buf_t, b_iodone_work);
Dave Chinner1813dd62012-11-14 17:54:40 +11001022 bool read = !!(bp->b_flags & XBF_READ);
1023
1024 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
Dave Chinnerd5929de2013-02-27 13:25:54 +11001025
1026 /* only validate buffers that were read without errors */
1027 if (read && bp->b_ops && !bp->b_error && (bp->b_flags & XBF_DONE))
Dave Chinner1813dd62012-11-14 17:54:40 +11001028 bp->b_ops->verify_read(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001029
Christoph Hellwig80f6c292010-08-18 05:29:11 -04001030 if (bp->b_iodone)
Nathan Scottce8e9222006-01-11 15:39:08 +11001031 (*(bp->b_iodone))(bp);
1032 else if (bp->b_flags & XBF_ASYNC)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001033 xfs_buf_relse(bp);
Dave Chinner1813dd62012-11-14 17:54:40 +11001034 else {
1035 ASSERT(read && bp->b_ops);
1036 complete(&bp->b_iowait);
1037 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001038}
1039
1040void
Nathan Scottce8e9222006-01-11 15:39:08 +11001041xfs_buf_ioend(
Dave Chinner1813dd62012-11-14 17:54:40 +11001042 struct xfs_buf *bp,
1043 int schedule)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001044{
Dave Chinner1813dd62012-11-14 17:54:40 +11001045 bool read = !!(bp->b_flags & XBF_READ);
1046
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001047 trace_xfs_buf_iodone(bp, _RET_IP_);
1048
Nathan Scottce8e9222006-01-11 15:39:08 +11001049 if (bp->b_error == 0)
1050 bp->b_flags |= XBF_DONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001051
Dave Chinner1813dd62012-11-14 17:54:40 +11001052 if (bp->b_iodone || (read && bp->b_ops) || (bp->b_flags & XBF_ASYNC)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001053 if (schedule) {
David Howellsc4028952006-11-22 14:57:56 +00001054 INIT_WORK(&bp->b_iodone_work, xfs_buf_iodone_work);
Nathan Scottce8e9222006-01-11 15:39:08 +11001055 queue_work(xfslogd_workqueue, &bp->b_iodone_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001056 } else {
David Howellsc4028952006-11-22 14:57:56 +00001057 xfs_buf_iodone_work(&bp->b_iodone_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001058 }
1059 } else {
Dave Chinner1813dd62012-11-14 17:54:40 +11001060 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
David Chinnerb4dd3302008-08-13 16:36:11 +10001061 complete(&bp->b_iowait);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001062 }
1063}
1064
Linus Torvalds1da177e2005-04-16 15:20:36 -07001065void
Nathan Scottce8e9222006-01-11 15:39:08 +11001066xfs_buf_ioerror(
1067 xfs_buf_t *bp,
1068 int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001069{
1070 ASSERT(error >= 0 && error <= 0xffff);
Nathan Scottce8e9222006-01-11 15:39:08 +11001071 bp->b_error = (unsigned short)error;
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001072 trace_xfs_buf_ioerror(bp, error, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001073}
1074
Christoph Hellwig901796a2011-10-10 16:52:49 +00001075void
1076xfs_buf_ioerror_alert(
1077 struct xfs_buf *bp,
1078 const char *func)
1079{
1080 xfs_alert(bp->b_target->bt_mount,
Dave Chinneraa0e8832012-04-23 15:58:52 +10001081"metadata I/O error: block 0x%llx (\"%s\") error %d numblks %d",
1082 (__uint64_t)XFS_BUF_ADDR(bp), func, bp->b_error, bp->b_length);
Christoph Hellwig901796a2011-10-10 16:52:49 +00001083}
1084
Christoph Hellwig4e234712010-01-13 22:17:56 +00001085/*
1086 * Called when we want to stop a buffer from getting written or read.
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +00001087 * We attach the EIO error, muck with its flags, and call xfs_buf_ioend
Christoph Hellwig4e234712010-01-13 22:17:56 +00001088 * so that the proper iodone callbacks get called.
1089 */
1090STATIC int
1091xfs_bioerror(
1092 xfs_buf_t *bp)
1093{
1094#ifdef XFSERRORDEBUG
1095 ASSERT(XFS_BUF_ISREAD(bp) || bp->b_iodone);
1096#endif
1097
1098 /*
1099 * No need to wait until the buffer is unpinned, we aren't flushing it.
1100 */
Chandra Seetharaman5a52c2a582011-07-22 23:39:51 +00001101 xfs_buf_ioerror(bp, EIO);
Christoph Hellwig4e234712010-01-13 22:17:56 +00001102
1103 /*
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +00001104 * We're calling xfs_buf_ioend, so delete XBF_DONE flag.
Christoph Hellwig4e234712010-01-13 22:17:56 +00001105 */
1106 XFS_BUF_UNREAD(bp);
Christoph Hellwig4e234712010-01-13 22:17:56 +00001107 XFS_BUF_UNDONE(bp);
Christoph Hellwigc867cb62011-10-10 16:52:46 +00001108 xfs_buf_stale(bp);
Christoph Hellwig4e234712010-01-13 22:17:56 +00001109
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +00001110 xfs_buf_ioend(bp, 0);
Christoph Hellwig4e234712010-01-13 22:17:56 +00001111
1112 return EIO;
1113}
1114
1115/*
1116 * Same as xfs_bioerror, except that we are releasing the buffer
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +00001117 * here ourselves, and avoiding the xfs_buf_ioend call.
Christoph Hellwig4e234712010-01-13 22:17:56 +00001118 * This is meant for userdata errors; metadata bufs come with
1119 * iodone functions attached, so that we can track down errors.
1120 */
1121STATIC int
1122xfs_bioerror_relse(
1123 struct xfs_buf *bp)
1124{
Chandra Seetharamaned432332011-07-22 23:39:39 +00001125 int64_t fl = bp->b_flags;
Christoph Hellwig4e234712010-01-13 22:17:56 +00001126 /*
1127 * No need to wait until the buffer is unpinned.
1128 * We aren't flushing it.
1129 *
1130 * chunkhold expects B_DONE to be set, whether
1131 * we actually finish the I/O or not. We don't want to
1132 * change that interface.
1133 */
1134 XFS_BUF_UNREAD(bp);
Christoph Hellwig4e234712010-01-13 22:17:56 +00001135 XFS_BUF_DONE(bp);
Christoph Hellwigc867cb62011-10-10 16:52:46 +00001136 xfs_buf_stale(bp);
Christoph Hellwigcb669ca2011-07-13 13:43:49 +02001137 bp->b_iodone = NULL;
Christoph Hellwig0cadda12010-01-19 09:56:44 +00001138 if (!(fl & XBF_ASYNC)) {
Christoph Hellwig4e234712010-01-13 22:17:56 +00001139 /*
1140 * Mark b_error and B_ERROR _both_.
1141 * Lot's of chunkcache code assumes that.
1142 * There's no reason to mark error for
1143 * ASYNC buffers.
1144 */
Chandra Seetharaman5a52c2a582011-07-22 23:39:51 +00001145 xfs_buf_ioerror(bp, EIO);
Christoph Hellwig5fde0322011-10-10 16:52:44 +00001146 complete(&bp->b_iowait);
Christoph Hellwig4e234712010-01-13 22:17:56 +00001147 } else {
1148 xfs_buf_relse(bp);
1149 }
1150
1151 return EIO;
1152}
1153
Christoph Hellwiga2dcf5d2012-07-13 02:24:10 -04001154STATIC int
Christoph Hellwig4e234712010-01-13 22:17:56 +00001155xfs_bdstrat_cb(
1156 struct xfs_buf *bp)
1157{
Dave Chinnerebad8612010-09-22 10:47:20 +10001158 if (XFS_FORCED_SHUTDOWN(bp->b_target->bt_mount)) {
Christoph Hellwig4e234712010-01-13 22:17:56 +00001159 trace_xfs_bdstrat_shut(bp, _RET_IP_);
1160 /*
1161 * Metadata write that didn't get logged but
1162 * written delayed anyway. These aren't associated
1163 * with a transaction, and can be ignored.
1164 */
1165 if (!bp->b_iodone && !XFS_BUF_ISREAD(bp))
1166 return xfs_bioerror_relse(bp);
1167 else
1168 return xfs_bioerror(bp);
1169 }
1170
1171 xfs_buf_iorequest(bp);
1172 return 0;
1173}
1174
Christoph Hellwiga2dcf5d2012-07-13 02:24:10 -04001175int
1176xfs_bwrite(
1177 struct xfs_buf *bp)
1178{
1179 int error;
1180
1181 ASSERT(xfs_buf_islocked(bp));
1182
1183 bp->b_flags |= XBF_WRITE;
1184 bp->b_flags &= ~(XBF_ASYNC | XBF_READ | _XBF_DELWRI_Q);
1185
1186 xfs_bdstrat_cb(bp);
1187
1188 error = xfs_buf_iowait(bp);
1189 if (error) {
1190 xfs_force_shutdown(bp->b_target->bt_mount,
1191 SHUTDOWN_META_IO_ERROR);
1192 }
1193 return error;
1194}
1195
Christoph Hellwig4e234712010-01-13 22:17:56 +00001196/*
1197 * Wrapper around bdstrat so that we can stop data from going to disk in case
1198 * we are shutting down the filesystem. Typically user data goes thru this
1199 * path; one of the exceptions is the superblock.
1200 */
1201void
1202xfsbdstrat(
1203 struct xfs_mount *mp,
1204 struct xfs_buf *bp)
1205{
1206 if (XFS_FORCED_SHUTDOWN(mp)) {
1207 trace_xfs_bdstrat_shut(bp, _RET_IP_);
1208 xfs_bioerror_relse(bp);
1209 return;
1210 }
1211
1212 xfs_buf_iorequest(bp);
1213}
1214
Christoph Hellwigb8f82a42009-11-14 16:17:22 +00001215STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001216_xfs_buf_ioend(
1217 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001218 int schedule)
1219{
Dave Chinner0e6e8472011-03-26 09:16:45 +11001220 if (atomic_dec_and_test(&bp->b_io_remaining) == 1)
Nathan Scottce8e9222006-01-11 15:39:08 +11001221 xfs_buf_ioend(bp, schedule);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001222}
1223
Al Viro782e3b32007-10-12 07:17:47 +01001224STATIC void
Nathan Scottce8e9222006-01-11 15:39:08 +11001225xfs_buf_bio_end_io(
Linus Torvalds1da177e2005-04-16 15:20:36 -07001226 struct bio *bio,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001227 int error)
1228{
Nathan Scottce8e9222006-01-11 15:39:08 +11001229 xfs_buf_t *bp = (xfs_buf_t *)bio->bi_private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001230
Dave Chinner37eb17e2012-11-12 22:09:46 +11001231 /*
1232 * don't overwrite existing errors - otherwise we can lose errors on
1233 * buffers that require multiple bios to complete.
1234 */
1235 if (!bp->b_error)
1236 xfs_buf_ioerror(bp, -error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001237
Dave Chinner37eb17e2012-11-12 22:09:46 +11001238 if (!bp->b_error && xfs_buf_is_vmapped(bp) && (bp->b_flags & XBF_READ))
James Bottomley73c77e22010-01-25 11:42:24 -06001239 invalidate_kernel_vmap_range(bp->b_addr, xfs_buf_vmap_len(bp));
1240
Nathan Scottce8e9222006-01-11 15:39:08 +11001241 _xfs_buf_ioend(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001242 bio_put(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001243}
1244
Dave Chinner3e85c862012-06-22 18:50:09 +10001245static void
1246xfs_buf_ioapply_map(
1247 struct xfs_buf *bp,
1248 int map,
1249 int *buf_offset,
1250 int *count,
1251 int rw)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001252{
Dave Chinner3e85c862012-06-22 18:50:09 +10001253 int page_index;
1254 int total_nr_pages = bp->b_page_count;
1255 int nr_pages;
1256 struct bio *bio;
1257 sector_t sector = bp->b_maps[map].bm_bn;
1258 int size;
1259 int offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001260
Nathan Scottce8e9222006-01-11 15:39:08 +11001261 total_nr_pages = bp->b_page_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001262
Dave Chinner3e85c862012-06-22 18:50:09 +10001263 /* skip the pages in the buffer before the start offset */
1264 page_index = 0;
1265 offset = *buf_offset;
1266 while (offset >= PAGE_SIZE) {
1267 page_index++;
1268 offset -= PAGE_SIZE;
Christoph Hellwigf538d4d2005-11-02 10:26:59 +11001269 }
1270
Dave Chinner3e85c862012-06-22 18:50:09 +10001271 /*
1272 * Limit the IO size to the length of the current vector, and update the
1273 * remaining IO count for the next time around.
1274 */
1275 size = min_t(int, BBTOB(bp->b_maps[map].bm_len), *count);
1276 *count -= size;
1277 *buf_offset += size;
Christoph Hellwig34951f52011-07-26 15:06:44 +00001278
Linus Torvalds1da177e2005-04-16 15:20:36 -07001279next_chunk:
Nathan Scottce8e9222006-01-11 15:39:08 +11001280 atomic_inc(&bp->b_io_remaining);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001281 nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1282 if (nr_pages > total_nr_pages)
1283 nr_pages = total_nr_pages;
1284
1285 bio = bio_alloc(GFP_NOIO, nr_pages);
Nathan Scottce8e9222006-01-11 15:39:08 +11001286 bio->bi_bdev = bp->b_target->bt_bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001287 bio->bi_sector = sector;
Nathan Scottce8e9222006-01-11 15:39:08 +11001288 bio->bi_end_io = xfs_buf_bio_end_io;
1289 bio->bi_private = bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001290
Dave Chinner0e6e8472011-03-26 09:16:45 +11001291
Dave Chinner3e85c862012-06-22 18:50:09 +10001292 for (; size && nr_pages; nr_pages--, page_index++) {
Dave Chinner0e6e8472011-03-26 09:16:45 +11001293 int rbytes, nbytes = PAGE_SIZE - offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001294
1295 if (nbytes > size)
1296 nbytes = size;
1297
Dave Chinner3e85c862012-06-22 18:50:09 +10001298 rbytes = bio_add_page(bio, bp->b_pages[page_index], nbytes,
1299 offset);
Nathan Scottce8e9222006-01-11 15:39:08 +11001300 if (rbytes < nbytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001301 break;
1302
1303 offset = 0;
Dave Chinneraa0e8832012-04-23 15:58:52 +10001304 sector += BTOBB(nbytes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001305 size -= nbytes;
1306 total_nr_pages--;
1307 }
1308
Linus Torvalds1da177e2005-04-16 15:20:36 -07001309 if (likely(bio->bi_size)) {
James Bottomley73c77e22010-01-25 11:42:24 -06001310 if (xfs_buf_is_vmapped(bp)) {
1311 flush_kernel_vmap_range(bp->b_addr,
1312 xfs_buf_vmap_len(bp));
1313 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001314 submit_bio(rw, bio);
1315 if (size)
1316 goto next_chunk;
1317 } else {
Dave Chinner37eb17e2012-11-12 22:09:46 +11001318 /*
1319 * This is guaranteed not to be the last io reference count
1320 * because the caller (xfs_buf_iorequest) holds a count itself.
1321 */
1322 atomic_dec(&bp->b_io_remaining);
Nathan Scottce8e9222006-01-11 15:39:08 +11001323 xfs_buf_ioerror(bp, EIO);
Dave Chinnerec53d1d2010-07-20 17:52:59 +10001324 bio_put(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001325 }
Dave Chinner3e85c862012-06-22 18:50:09 +10001326
1327}
1328
1329STATIC void
1330_xfs_buf_ioapply(
1331 struct xfs_buf *bp)
1332{
1333 struct blk_plug plug;
1334 int rw;
1335 int offset;
1336 int size;
1337 int i;
1338
Dave Chinnerc163f9a2013-03-12 23:30:34 +11001339 /*
1340 * Make sure we capture only current IO errors rather than stale errors
1341 * left over from previous use of the buffer (e.g. failed readahead).
1342 */
1343 bp->b_error = 0;
1344
Dave Chinner3e85c862012-06-22 18:50:09 +10001345 if (bp->b_flags & XBF_WRITE) {
1346 if (bp->b_flags & XBF_SYNCIO)
1347 rw = WRITE_SYNC;
1348 else
1349 rw = WRITE;
1350 if (bp->b_flags & XBF_FUA)
1351 rw |= REQ_FUA;
1352 if (bp->b_flags & XBF_FLUSH)
1353 rw |= REQ_FLUSH;
Dave Chinner1813dd62012-11-14 17:54:40 +11001354
1355 /*
1356 * Run the write verifier callback function if it exists. If
1357 * this function fails it will mark the buffer with an error and
1358 * the IO should not be dispatched.
1359 */
1360 if (bp->b_ops) {
1361 bp->b_ops->verify_write(bp);
1362 if (bp->b_error) {
1363 xfs_force_shutdown(bp->b_target->bt_mount,
1364 SHUTDOWN_CORRUPT_INCORE);
1365 return;
1366 }
1367 }
Dave Chinner3e85c862012-06-22 18:50:09 +10001368 } else if (bp->b_flags & XBF_READ_AHEAD) {
1369 rw = READA;
1370 } else {
1371 rw = READ;
1372 }
1373
1374 /* we only use the buffer cache for meta-data */
1375 rw |= REQ_META;
1376
1377 /*
1378 * Walk all the vectors issuing IO on them. Set up the initial offset
1379 * into the buffer and the desired IO size before we start -
1380 * _xfs_buf_ioapply_vec() will modify them appropriately for each
1381 * subsequent call.
1382 */
1383 offset = bp->b_offset;
1384 size = BBTOB(bp->b_io_length);
1385 blk_start_plug(&plug);
1386 for (i = 0; i < bp->b_map_count; i++) {
1387 xfs_buf_ioapply_map(bp, i, &offset, &size, rw);
1388 if (bp->b_error)
1389 break;
1390 if (size <= 0)
1391 break; /* all done */
1392 }
1393 blk_finish_plug(&plug);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001394}
1395
Dave Chinner0e95f192012-04-23 15:58:46 +10001396void
Nathan Scottce8e9222006-01-11 15:39:08 +11001397xfs_buf_iorequest(
1398 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001399{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001400 trace_xfs_buf_iorequest(bp, _RET_IP_);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001401
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001402 ASSERT(!(bp->b_flags & _XBF_DELWRI_Q));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001403
Christoph Hellwig375ec69d2011-08-23 08:28:03 +00001404 if (bp->b_flags & XBF_WRITE)
Nathan Scottce8e9222006-01-11 15:39:08 +11001405 xfs_buf_wait_unpin(bp);
Nathan Scottce8e9222006-01-11 15:39:08 +11001406 xfs_buf_hold(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001407
1408 /* Set the count to 1 initially, this will stop an I/O
1409 * completion callout which happens before we have started
Nathan Scottce8e9222006-01-11 15:39:08 +11001410 * all the I/O from calling xfs_buf_ioend too early.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001411 */
Nathan Scottce8e9222006-01-11 15:39:08 +11001412 atomic_set(&bp->b_io_remaining, 1);
1413 _xfs_buf_ioapply(bp);
Christoph Hellwig08023d62012-07-02 06:00:04 -04001414 _xfs_buf_ioend(bp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001415
Nathan Scottce8e9222006-01-11 15:39:08 +11001416 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001417}
1418
1419/*
Dave Chinner0e95f192012-04-23 15:58:46 +10001420 * Waits for I/O to complete on the buffer supplied. It returns immediately if
1421 * no I/O is pending or there is already a pending error on the buffer. It
1422 * returns the I/O error code, if any, or 0 if there was no error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001423 */
1424int
Nathan Scottce8e9222006-01-11 15:39:08 +11001425xfs_buf_iowait(
1426 xfs_buf_t *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001427{
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001428 trace_xfs_buf_iowait(bp, _RET_IP_);
1429
Dave Chinner0e95f192012-04-23 15:58:46 +10001430 if (!bp->b_error)
1431 wait_for_completion(&bp->b_iowait);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001432
1433 trace_xfs_buf_iowait_done(bp, _RET_IP_);
Nathan Scottce8e9222006-01-11 15:39:08 +11001434 return bp->b_error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001435}
1436
Nathan Scottce8e9222006-01-11 15:39:08 +11001437xfs_caddr_t
1438xfs_buf_offset(
1439 xfs_buf_t *bp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001440 size_t offset)
1441{
1442 struct page *page;
1443
Dave Chinner611c9942012-04-23 15:59:07 +10001444 if (bp->b_addr)
Chandra Seetharaman62926042011-07-22 23:40:15 +00001445 return bp->b_addr + offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001446
Nathan Scottce8e9222006-01-11 15:39:08 +11001447 offset += bp->b_offset;
Dave Chinner0e6e8472011-03-26 09:16:45 +11001448 page = bp->b_pages[offset >> PAGE_SHIFT];
1449 return (xfs_caddr_t)page_address(page) + (offset & (PAGE_SIZE-1));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001450}
1451
1452/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001453 * Move data into or out of a buffer.
1454 */
1455void
Nathan Scottce8e9222006-01-11 15:39:08 +11001456xfs_buf_iomove(
1457 xfs_buf_t *bp, /* buffer to process */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001458 size_t boff, /* starting buffer offset */
1459 size_t bsize, /* length to copy */
Dave Chinnerb9c48642010-01-20 10:47:39 +11001460 void *data, /* data address */
Nathan Scottce8e9222006-01-11 15:39:08 +11001461 xfs_buf_rw_t mode) /* read/write/zero flag */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001462{
Dave Chinner795cac72012-04-23 15:58:53 +10001463 size_t bend;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001464
1465 bend = boff + bsize;
1466 while (boff < bend) {
Dave Chinner795cac72012-04-23 15:58:53 +10001467 struct page *page;
1468 int page_index, page_offset, csize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001469
Dave Chinner795cac72012-04-23 15:58:53 +10001470 page_index = (boff + bp->b_offset) >> PAGE_SHIFT;
1471 page_offset = (boff + bp->b_offset) & ~PAGE_MASK;
1472 page = bp->b_pages[page_index];
1473 csize = min_t(size_t, PAGE_SIZE - page_offset,
1474 BBTOB(bp->b_io_length) - boff);
1475
1476 ASSERT((csize + page_offset) <= PAGE_SIZE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001477
1478 switch (mode) {
Nathan Scottce8e9222006-01-11 15:39:08 +11001479 case XBRW_ZERO:
Dave Chinner795cac72012-04-23 15:58:53 +10001480 memset(page_address(page) + page_offset, 0, csize);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001481 break;
Nathan Scottce8e9222006-01-11 15:39:08 +11001482 case XBRW_READ:
Dave Chinner795cac72012-04-23 15:58:53 +10001483 memcpy(data, page_address(page) + page_offset, csize);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001484 break;
Nathan Scottce8e9222006-01-11 15:39:08 +11001485 case XBRW_WRITE:
Dave Chinner795cac72012-04-23 15:58:53 +10001486 memcpy(page_address(page) + page_offset, data, csize);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001487 }
1488
1489 boff += csize;
1490 data += csize;
1491 }
1492}
1493
1494/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001495 * Handling of buffer targets (buftargs).
Linus Torvalds1da177e2005-04-16 15:20:36 -07001496 */
1497
1498/*
Dave Chinner430cbeb2010-12-02 16:30:55 +11001499 * Wait for any bufs with callbacks that have been submitted but have not yet
1500 * returned. These buffers will have an elevated hold count, so wait on those
1501 * while freeing all the buffers only held by the LRU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001502 */
1503void
1504xfs_wait_buftarg(
Dave Chinner74f75a02010-09-24 19:59:04 +10001505 struct xfs_buftarg *btp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001506{
Dave Chinner430cbeb2010-12-02 16:30:55 +11001507 struct xfs_buf *bp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001508
Dave Chinner430cbeb2010-12-02 16:30:55 +11001509restart:
1510 spin_lock(&btp->bt_lru_lock);
1511 while (!list_empty(&btp->bt_lru)) {
1512 bp = list_first_entry(&btp->bt_lru, struct xfs_buf, b_lru);
1513 if (atomic_read(&bp->b_hold) > 1) {
Dave Chinner3b190342013-01-21 23:53:55 +11001514 trace_xfs_buf_wait_buftarg(bp, _RET_IP_);
1515 list_move_tail(&bp->b_lru, &btp->bt_lru);
Dave Chinner430cbeb2010-12-02 16:30:55 +11001516 spin_unlock(&btp->bt_lru_lock);
Dave Chinner26af6552010-09-22 10:47:20 +10001517 delay(100);
Dave Chinner430cbeb2010-12-02 16:30:55 +11001518 goto restart;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001519 }
Dave Chinner430cbeb2010-12-02 16:30:55 +11001520 /*
Paul Bolle90802ed2011-12-05 13:00:34 +01001521 * clear the LRU reference count so the buffer doesn't get
Dave Chinner430cbeb2010-12-02 16:30:55 +11001522 * ignored in xfs_buf_rele().
1523 */
1524 atomic_set(&bp->b_lru_ref, 0);
1525 spin_unlock(&btp->bt_lru_lock);
1526 xfs_buf_rele(bp);
1527 spin_lock(&btp->bt_lru_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001528 }
Dave Chinner430cbeb2010-12-02 16:30:55 +11001529 spin_unlock(&btp->bt_lru_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001530}
1531
Dave Chinnerff57ab22010-11-30 17:27:57 +11001532int
1533xfs_buftarg_shrink(
1534 struct shrinker *shrink,
Ying Han1495f232011-05-24 17:12:27 -07001535 struct shrink_control *sc)
David Chinnera6867a62006-01-11 15:37:58 +11001536{
Dave Chinnerff57ab22010-11-30 17:27:57 +11001537 struct xfs_buftarg *btp = container_of(shrink,
1538 struct xfs_buftarg, bt_shrinker);
Dave Chinner430cbeb2010-12-02 16:30:55 +11001539 struct xfs_buf *bp;
Ying Han1495f232011-05-24 17:12:27 -07001540 int nr_to_scan = sc->nr_to_scan;
Dave Chinner430cbeb2010-12-02 16:30:55 +11001541 LIST_HEAD(dispose);
1542
1543 if (!nr_to_scan)
1544 return btp->bt_lru_nr;
1545
1546 spin_lock(&btp->bt_lru_lock);
1547 while (!list_empty(&btp->bt_lru)) {
1548 if (nr_to_scan-- <= 0)
1549 break;
1550
1551 bp = list_first_entry(&btp->bt_lru, struct xfs_buf, b_lru);
1552
1553 /*
1554 * Decrement the b_lru_ref count unless the value is already
1555 * zero. If the value is already zero, we need to reclaim the
1556 * buffer, otherwise it gets another trip through the LRU.
1557 */
1558 if (!atomic_add_unless(&bp->b_lru_ref, -1, 0)) {
1559 list_move_tail(&bp->b_lru, &btp->bt_lru);
1560 continue;
1561 }
1562
1563 /*
1564 * remove the buffer from the LRU now to avoid needing another
1565 * lock round trip inside xfs_buf_rele().
1566 */
1567 list_move(&bp->b_lru, &dispose);
1568 btp->bt_lru_nr--;
Carlos Maiolino6fb8a902012-08-10 15:01:51 -03001569 bp->b_lru_flags |= _XBF_LRU_DISPOSE;
Dave Chinnerff57ab22010-11-30 17:27:57 +11001570 }
Dave Chinner430cbeb2010-12-02 16:30:55 +11001571 spin_unlock(&btp->bt_lru_lock);
1572
1573 while (!list_empty(&dispose)) {
1574 bp = list_first_entry(&dispose, struct xfs_buf, b_lru);
1575 list_del_init(&bp->b_lru);
1576 xfs_buf_rele(bp);
1577 }
1578
1579 return btp->bt_lru_nr;
David Chinnera6867a62006-01-11 15:37:58 +11001580}
1581
Linus Torvalds1da177e2005-04-16 15:20:36 -07001582void
1583xfs_free_buftarg(
Christoph Hellwigb7963132009-03-03 14:48:37 -05001584 struct xfs_mount *mp,
1585 struct xfs_buftarg *btp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001586{
Dave Chinnerff57ab22010-11-30 17:27:57 +11001587 unregister_shrinker(&btp->bt_shrinker);
1588
Christoph Hellwigb7963132009-03-03 14:48:37 -05001589 if (mp->m_flags & XFS_MOUNT_BARRIER)
1590 xfs_blkdev_issue_flush(btp);
David Chinnera6867a62006-01-11 15:37:58 +11001591
Denys Vlasenkof0e2d932008-05-19 16:31:57 +10001592 kmem_free(btp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001593}
1594
Linus Torvalds1da177e2005-04-16 15:20:36 -07001595STATIC int
1596xfs_setsize_buftarg_flags(
1597 xfs_buftarg_t *btp,
1598 unsigned int blocksize,
1599 unsigned int sectorsize,
1600 int verbose)
1601{
Nathan Scottce8e9222006-01-11 15:39:08 +11001602 btp->bt_bsize = blocksize;
1603 btp->bt_sshift = ffs(sectorsize) - 1;
1604 btp->bt_smask = sectorsize - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001605
Nathan Scottce8e9222006-01-11 15:39:08 +11001606 if (set_blocksize(btp->bt_bdev, sectorsize)) {
Christoph Hellwig02b102d2011-10-10 16:52:51 +00001607 char name[BDEVNAME_SIZE];
1608
1609 bdevname(btp->bt_bdev, name);
1610
Dave Chinner4f107002011-03-07 10:00:35 +11001611 xfs_warn(btp->bt_mount,
1612 "Cannot set_blocksize to %u on device %s\n",
Christoph Hellwig02b102d2011-10-10 16:52:51 +00001613 sectorsize, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001614 return EINVAL;
1615 }
1616
Linus Torvalds1da177e2005-04-16 15:20:36 -07001617 return 0;
1618}
1619
1620/*
Nathan Scottce8e9222006-01-11 15:39:08 +11001621 * When allocating the initial buffer target we have not yet
1622 * read in the superblock, so don't know what sized sectors
1623 * are being used is at this early stage. Play safe.
1624 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001625STATIC int
1626xfs_setsize_buftarg_early(
1627 xfs_buftarg_t *btp,
1628 struct block_device *bdev)
1629{
1630 return xfs_setsize_buftarg_flags(btp,
Dave Chinner0e6e8472011-03-26 09:16:45 +11001631 PAGE_SIZE, bdev_logical_block_size(bdev), 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001632}
1633
1634int
1635xfs_setsize_buftarg(
1636 xfs_buftarg_t *btp,
1637 unsigned int blocksize,
1638 unsigned int sectorsize)
1639{
1640 return xfs_setsize_buftarg_flags(btp, blocksize, sectorsize, 1);
1641}
1642
Linus Torvalds1da177e2005-04-16 15:20:36 -07001643xfs_buftarg_t *
1644xfs_alloc_buftarg(
Dave Chinnerebad8612010-09-22 10:47:20 +10001645 struct xfs_mount *mp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001646 struct block_device *bdev,
Jan Engelhardte2a07812010-03-23 09:52:55 +11001647 int external,
1648 const char *fsname)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001649{
1650 xfs_buftarg_t *btp;
1651
1652 btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
1653
Dave Chinnerebad8612010-09-22 10:47:20 +10001654 btp->bt_mount = mp;
Nathan Scottce8e9222006-01-11 15:39:08 +11001655 btp->bt_dev = bdev->bd_dev;
1656 btp->bt_bdev = bdev;
Dave Chinner0e6e8472011-03-26 09:16:45 +11001657 btp->bt_bdi = blk_get_backing_dev_info(bdev);
1658 if (!btp->bt_bdi)
1659 goto error;
1660
Dave Chinner430cbeb2010-12-02 16:30:55 +11001661 INIT_LIST_HEAD(&btp->bt_lru);
1662 spin_lock_init(&btp->bt_lru_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001663 if (xfs_setsize_buftarg_early(btp, bdev))
1664 goto error;
Dave Chinnerff57ab22010-11-30 17:27:57 +11001665 btp->bt_shrinker.shrink = xfs_buftarg_shrink;
1666 btp->bt_shrinker.seeks = DEFAULT_SEEKS;
1667 register_shrinker(&btp->bt_shrinker);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001668 return btp;
1669
1670error:
Denys Vlasenkof0e2d932008-05-19 16:31:57 +10001671 kmem_free(btp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001672 return NULL;
1673}
1674
Linus Torvalds1da177e2005-04-16 15:20:36 -07001675/*
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001676 * Add a buffer to the delayed write list.
1677 *
1678 * This queues a buffer for writeout if it hasn't already been. Note that
1679 * neither this routine nor the buffer list submission functions perform
1680 * any internal synchronization. It is expected that the lists are thread-local
1681 * to the callers.
1682 *
1683 * Returns true if we queued up the buffer, or false if it already had
1684 * been on the buffer list.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001685 */
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001686bool
Nathan Scottce8e9222006-01-11 15:39:08 +11001687xfs_buf_delwri_queue(
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001688 struct xfs_buf *bp,
1689 struct list_head *list)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001690{
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001691 ASSERT(xfs_buf_islocked(bp));
1692 ASSERT(!(bp->b_flags & XBF_READ));
1693
1694 /*
1695 * If the buffer is already marked delwri it already is queued up
1696 * by someone else for imediate writeout. Just ignore it in that
1697 * case.
1698 */
1699 if (bp->b_flags & _XBF_DELWRI_Q) {
1700 trace_xfs_buf_delwri_queued(bp, _RET_IP_);
1701 return false;
1702 }
David Chinnera6867a62006-01-11 15:37:58 +11001703
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001704 trace_xfs_buf_delwri_queue(bp, _RET_IP_);
1705
Dave Chinnerd808f612010-02-02 10:13:42 +11001706 /*
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001707 * If a buffer gets written out synchronously or marked stale while it
1708 * is on a delwri list we lazily remove it. To do this, the other party
1709 * clears the _XBF_DELWRI_Q flag but otherwise leaves the buffer alone.
1710 * It remains referenced and on the list. In a rare corner case it
1711 * might get readded to a delwri list after the synchronous writeout, in
1712 * which case we need just need to re-add the flag here.
Dave Chinnerd808f612010-02-02 10:13:42 +11001713 */
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001714 bp->b_flags |= _XBF_DELWRI_Q;
1715 if (list_empty(&bp->b_list)) {
1716 atomic_inc(&bp->b_hold);
1717 list_add_tail(&bp->b_list, list);
David Chinner585e6d82007-02-10 18:32:29 +11001718 }
David Chinner585e6d82007-02-10 18:32:29 +11001719
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001720 return true;
David Chinner585e6d82007-02-10 18:32:29 +11001721}
1722
Dave Chinner089716a2010-01-26 15:13:25 +11001723/*
1724 * Compare function is more complex than it needs to be because
1725 * the return value is only 32 bits and we are doing comparisons
1726 * on 64 bit values
1727 */
1728static int
1729xfs_buf_cmp(
1730 void *priv,
1731 struct list_head *a,
1732 struct list_head *b)
1733{
1734 struct xfs_buf *ap = container_of(a, struct xfs_buf, b_list);
1735 struct xfs_buf *bp = container_of(b, struct xfs_buf, b_list);
1736 xfs_daddr_t diff;
1737
Mark Tinguelyf4b42422012-12-04 17:18:02 -06001738 diff = ap->b_maps[0].bm_bn - bp->b_maps[0].bm_bn;
Dave Chinner089716a2010-01-26 15:13:25 +11001739 if (diff < 0)
1740 return -1;
1741 if (diff > 0)
1742 return 1;
1743 return 0;
1744}
1745
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001746static int
1747__xfs_buf_delwri_submit(
1748 struct list_head *buffer_list,
1749 struct list_head *io_list,
1750 bool wait)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001751{
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001752 struct blk_plug plug;
1753 struct xfs_buf *bp, *n;
1754 int pinned = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001755
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001756 list_for_each_entry_safe(bp, n, buffer_list, b_list) {
1757 if (!wait) {
1758 if (xfs_buf_ispinned(bp)) {
1759 pinned++;
1760 continue;
1761 }
1762 if (!xfs_buf_trylock(bp))
1763 continue;
1764 } else {
1765 xfs_buf_lock(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001766 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001767
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001768 /*
1769 * Someone else might have written the buffer synchronously or
1770 * marked it stale in the meantime. In that case only the
1771 * _XBF_DELWRI_Q flag got cleared, and we have to drop the
1772 * reference and remove it from the list here.
1773 */
1774 if (!(bp->b_flags & _XBF_DELWRI_Q)) {
1775 list_del_init(&bp->b_list);
1776 xfs_buf_relse(bp);
1777 continue;
1778 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001779
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001780 list_move_tail(&bp->b_list, io_list);
1781 trace_xfs_buf_delwri_split(bp, _RET_IP_);
1782 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001783
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001784 list_sort(NULL, io_list, xfs_buf_cmp);
Christoph Hellwiga1b7ea52011-03-30 11:05:09 +00001785
1786 blk_start_plug(&plug);
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001787 list_for_each_entry_safe(bp, n, io_list, b_list) {
1788 bp->b_flags &= ~(_XBF_DELWRI_Q | XBF_ASYNC);
1789 bp->b_flags |= XBF_WRITE;
1790
1791 if (!wait) {
1792 bp->b_flags |= XBF_ASYNC;
1793 list_del_init(&bp->b_list);
Dave Chinner089716a2010-01-26 15:13:25 +11001794 }
Christoph Hellwig939d7232010-07-20 17:51:16 +10001795 xfs_bdstrat_cb(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001796 }
Christoph Hellwiga1b7ea52011-03-30 11:05:09 +00001797 blk_finish_plug(&plug);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001798
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001799 return pinned;
1800}
Nathan Scottf07c2252006-09-28 10:52:15 +10001801
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001802/*
1803 * Write out a buffer list asynchronously.
1804 *
1805 * This will take the @buffer_list, write all non-locked and non-pinned buffers
1806 * out and not wait for I/O completion on any of the buffers. This interface
1807 * is only safely useable for callers that can track I/O completion by higher
1808 * level means, e.g. AIL pushing as the @buffer_list is consumed in this
1809 * function.
1810 */
1811int
1812xfs_buf_delwri_submit_nowait(
1813 struct list_head *buffer_list)
1814{
1815 LIST_HEAD (io_list);
1816 return __xfs_buf_delwri_submit(buffer_list, &io_list, false);
1817}
1818
1819/*
1820 * Write out a buffer list synchronously.
1821 *
1822 * This will take the @buffer_list, write all buffers out and wait for I/O
1823 * completion on all of the buffers. @buffer_list is consumed by the function,
1824 * so callers must have some other way of tracking buffers if they require such
1825 * functionality.
1826 */
1827int
1828xfs_buf_delwri_submit(
1829 struct list_head *buffer_list)
1830{
1831 LIST_HEAD (io_list);
1832 int error = 0, error2;
1833 struct xfs_buf *bp;
1834
1835 __xfs_buf_delwri_submit(buffer_list, &io_list, true);
1836
1837 /* Wait for IO to complete. */
1838 while (!list_empty(&io_list)) {
1839 bp = list_first_entry(&io_list, struct xfs_buf, b_list);
1840
1841 list_del_init(&bp->b_list);
1842 error2 = xfs_buf_iowait(bp);
1843 xfs_buf_relse(bp);
1844 if (!error)
1845 error = error2;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001846 }
1847
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001848 return error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001849}
1850
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001851int __init
Nathan Scottce8e9222006-01-11 15:39:08 +11001852xfs_buf_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001853{
Nathan Scott87582802006-03-14 13:18:19 +11001854 xfs_buf_zone = kmem_zone_init_flags(sizeof(xfs_buf_t), "xfs_buf",
1855 KM_ZONE_HWALIGN, NULL);
Nathan Scottce8e9222006-01-11 15:39:08 +11001856 if (!xfs_buf_zone)
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001857 goto out;
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001858
Dave Chinner51749e42010-09-08 09:00:22 +00001859 xfslogd_workqueue = alloc_workqueue("xfslogd",
Tejun Heo6370a6a2010-10-11 15:12:27 +02001860 WQ_MEM_RECLAIM | WQ_HIGHPRI, 1);
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001861 if (!xfslogd_workqueue)
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001862 goto out_free_buf_zone;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001863
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001864 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001865
Christoph Hellwig23ea4032005-06-21 15:14:01 +10001866 out_free_buf_zone:
Nathan Scottce8e9222006-01-11 15:39:08 +11001867 kmem_zone_destroy(xfs_buf_zone);
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001868 out:
Nathan Scott87582802006-03-14 13:18:19 +11001869 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001870}
1871
Linus Torvalds1da177e2005-04-16 15:20:36 -07001872void
Nathan Scottce8e9222006-01-11 15:39:08 +11001873xfs_buf_terminate(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001874{
Christoph Hellwig04d8b282005-11-02 10:15:05 +11001875 destroy_workqueue(xfslogd_workqueue);
Nathan Scottce8e9222006-01-11 15:39:08 +11001876 kmem_zone_destroy(xfs_buf_zone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001877}