block: manipulate bio->bi_flags through helpers
Some places use helpers now, others don't. We only have the 'is set'
helper, add helpers for setting and clearing flags too.
It was a bit of a mess of atomic vs non-atomic access. With
BIO_UPTODATE gone, we don't have any risk of concurrent access to the
flags. So relax the restriction and don't make any of them atomic. The
flags that do have serialization issues (reffed and chained), we
already handle those separately.
Signed-off-by: Jens Axboe <axboe@fb.com>
diff --git a/block/bio.c b/block/bio.c
index a23f489..911ae8f 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -311,7 +311,7 @@
*/
static inline void bio_inc_remaining(struct bio *bio)
{
- bio->bi_flags |= (1 << BIO_CHAIN);
+ bio_set_flag(bio, BIO_CHAIN);
smp_mb__before_atomic();
atomic_inc(&bio->__bi_remaining);
}
@@ -495,7 +495,7 @@
if (unlikely(!bvl))
goto err_free;
- bio->bi_flags |= 1 << BIO_OWNS_VEC;
+ bio_set_flag(bio, BIO_OWNS_VEC);
} else if (nr_iovecs) {
bvl = bio->bi_inline_vecs;
}
@@ -580,7 +580,7 @@
* so we don't set nor calculate new physical/hw segment counts here
*/
bio->bi_bdev = bio_src->bi_bdev;
- bio->bi_flags |= 1 << BIO_CLONED;
+ bio_set_flag(bio, BIO_CLONED);
bio->bi_rw = bio_src->bi_rw;
bio->bi_iter = bio_src->bi_iter;
bio->bi_io_vec = bio_src->bi_io_vec;
@@ -829,7 +829,7 @@
/* If we may be able to merge these biovecs, force a recount */
if (bio->bi_vcnt > 1 && (BIOVEC_PHYS_MERGEABLE(bvec-1, bvec)))
- bio->bi_flags &= ~(1 << BIO_SEG_VALID);
+ bio_clear_flag(bio, BIO_SEG_VALID);
done:
return len;
@@ -1390,7 +1390,7 @@
if (iter->type & WRITE)
bio->bi_rw |= REQ_WRITE;
- bio->bi_flags |= (1 << BIO_USER_MAPPED);
+ bio_set_flag(bio, BIO_USER_MAPPED);
/*
* subtle -- if __bio_map_user() ended up bouncing a bio,
@@ -1770,7 +1770,7 @@
BUG_ON(atomic_read(&bio->__bi_remaining) <= 0);
if (atomic_dec_and_test(&bio->__bi_remaining)) {
- clear_bit(BIO_CHAIN, &bio->bi_flags);
+ bio_clear_flag(bio, BIO_CHAIN);
return true;
}
@@ -1866,7 +1866,7 @@
if (offset == 0 && size == bio->bi_iter.bi_size)
return;
- clear_bit(BIO_SEG_VALID, &bio->bi_flags);
+ bio_clear_flag(bio, BIO_SEG_VALID);
bio_advance(bio, offset << 9);
diff --git a/block/blk-core.c b/block/blk-core.c
index 7ef15b9..d1796b5 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -146,7 +146,7 @@
bio->bi_error = error;
if (unlikely(rq->cmd_flags & REQ_QUIET))
- set_bit(BIO_QUIET, &bio->bi_flags);
+ bio_set_flag(bio, BIO_QUIET);
bio_advance(bio, nbytes);
diff --git a/block/blk-map.c b/block/blk-map.c
index 5fe1c30..2338416 100644
--- a/block/blk-map.c
+++ b/block/blk-map.c
@@ -94,7 +94,7 @@
return PTR_ERR(bio);
if (map_data && map_data->null_mapped)
- bio->bi_flags |= (1 << BIO_NULL_MAPPED);
+ bio_set_flag(bio, BIO_NULL_MAPPED);
if (bio->bi_iter.bi_size != iter->count) {
/*
diff --git a/block/blk-merge.c b/block/blk-merge.c
index 30a0d9f..a455b98 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -116,7 +116,7 @@
bio->bi_next = nxt;
}
- bio->bi_flags |= (1 << BIO_SEG_VALID);
+ bio_set_flag(bio, BIO_SEG_VALID);
}
EXPORT_SYMBOL(blk_recount_segments);
diff --git a/block/bounce.c b/block/bounce.c
index f4db245..2c310ea 100644
--- a/block/bounce.c
+++ b/block/bounce.c
@@ -186,7 +186,7 @@
if (!bdi_cap_stable_pages_required(&q->backing_dev_info))
return 0;
- return test_bit(BIO_SNAP_STABLE, &bio->bi_flags);
+ return bio_flagged(bio, BIO_SNAP_STABLE);
}
#else
static int must_snapshot_stable_pages(struct request_queue *q, struct bio *bio)
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
index 9aa7d1f..60d0a86 100644
--- a/drivers/md/raid1.c
+++ b/drivers/md/raid1.c
@@ -1157,7 +1157,7 @@
* non-zero, then it is the number of not-completed requests.
*/
bio->bi_phys_segments = 0;
- clear_bit(BIO_SEG_VALID, &bio->bi_flags);
+ bio_clear_flag(bio, BIO_SEG_VALID);
if (rw == READ) {
/*
@@ -2711,7 +2711,7 @@
/* remove last page from this bio */
bio->bi_vcnt--;
bio->bi_iter.bi_size -= len;
- __clear_bit(BIO_SEG_VALID, &bio->bi_flags);
+ bio_clear_flag(bio, BIO_SEG_VALID);
}
goto bio_full;
}
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c
index 929e9a2..316ff6f 100644
--- a/drivers/md/raid10.c
+++ b/drivers/md/raid10.c
@@ -1216,7 +1216,7 @@
* non-zero, then it is the number of not-completed requests.
*/
bio->bi_phys_segments = 0;
- clear_bit(BIO_SEG_VALID, &bio->bi_flags);
+ bio_clear_flag(bio, BIO_SEG_VALID);
if (rw == READ) {
/*
@@ -3353,7 +3353,7 @@
/* remove last page from this bio */
bio2->bi_vcnt--;
bio2->bi_iter.bi_size -= len;
- __clear_bit(BIO_SEG_VALID, &bio2->bi_flags);
+ bio_clear_flag(bio2, BIO_SEG_VALID);
}
goto bio_full;
}
@@ -4433,7 +4433,7 @@
/* Remove last page from this bio */
bio2->bi_vcnt--;
bio2->bi_iter.bi_size -= len;
- __clear_bit(BIO_SEG_VALID, &bio2->bi_flags);
+ bio_clear_flag(bio2, BIO_SEG_VALID);
}
goto bio_full;
}
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index 84d6eec..e3d4877 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -4850,7 +4850,7 @@
rcu_read_unlock();
raid_bio->bi_next = (void*)rdev;
align_bi->bi_bdev = rdev->bdev;
- __clear_bit(BIO_SEG_VALID, &align_bi->bi_flags);
+ bio_clear_flag(align_bi, BIO_SEG_VALID);
if (!bio_fits_rdev(align_bi) ||
is_badblock(rdev, align_bi->bi_iter.bi_sector,
diff --git a/fs/buffer.c b/fs/buffer.c
index 7a49bb8..7887bb4 100644
--- a/fs/buffer.c
+++ b/fs/buffer.c
@@ -2961,7 +2961,7 @@
{
struct buffer_head *bh = bio->bi_private;
- if (unlikely (test_bit(BIO_QUIET,&bio->bi_flags)))
+ if (unlikely(bio_flagged(bio, BIO_QUIET)))
set_bit(BH_Quiet, &bh->b_state);
bh->b_end_io(bh, !bio->bi_error);
diff --git a/include/linux/bio.h b/include/linux/bio.h
index 6b91817..986e6e19 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -304,6 +304,21 @@
atomic_set(&bio->__bi_cnt, count);
}
+static inline bool bio_flagged(struct bio *bio, unsigned int bit)
+{
+ return (bio->bi_flags & (1UL << bit)) != 0;
+}
+
+static inline void bio_set_flag(struct bio *bio, unsigned int bit)
+{
+ bio->bi_flags |= (1UL << bit);
+}
+
+static inline void bio_clear_flag(struct bio *bio, unsigned int bit)
+{
+ bio->bi_flags &= ~(1UL << bit);
+}
+
enum bip_flags {
BIP_BLOCK_INTEGRITY = 1 << 0, /* block layer owns integrity data */
BIP_MAPPED_INTEGRITY = 1 << 1, /* ref tag has been remapped */
diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index 6164fb8..a765a50 100644
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -129,8 +129,6 @@
#define BIO_RESET_BITS 13
#define BIO_OWNS_VEC 13 /* bio_free() should free bvec */
-#define bio_flagged(bio, flag) ((bio)->bi_flags & (1 << (flag)))
-
/*
* top 4 bits of bio flags indicate the pool this bio came from
*/