block: avoid duplicate calls to get_part() in disk stat code
get_part() is fairly expensive, as it O(N) loops over partitions
to find the right one. In lots of normal IO paths we end up looking
up the partition twice, to make matters even worse. Change the
stat add code to accept a passed in partition instead.
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
diff --git a/block/blk-core.c b/block/blk-core.c
index 1b7dddf..2987fe4 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -54,15 +54,16 @@
static void drive_stat_acct(struct request *rq, int new_io)
{
+ struct hd_struct *part;
int rw = rq_data_dir(rq);
if (!blk_fs_request(rq) || !rq->rq_disk)
return;
- if (!new_io) {
- __all_stat_inc(rq->rq_disk, merges[rw], rq->sector);
- } else {
- struct hd_struct *part = get_part(rq->rq_disk, rq->sector);
+ part = get_part(rq->rq_disk, rq->sector);
+ if (!new_io)
+ __all_stat_inc(rq->rq_disk, part, merges[rw], rq->sector);
+ else {
disk_round_stats(rq->rq_disk);
rq->rq_disk->in_flight++;
if (part) {
@@ -1538,10 +1539,11 @@
}
if (blk_fs_request(req) && req->rq_disk) {
+ struct hd_struct *part = get_part(req->rq_disk, req->sector);
const int rw = rq_data_dir(req);
- all_stat_add(req->rq_disk, sectors[rw],
- nr_bytes >> 9, req->sector);
+ all_stat_add(req->rq_disk, part, sectors[rw],
+ nr_bytes >> 9, req->sector);
}
total_bytes = bio_nbytes = 0;
@@ -1727,8 +1729,8 @@
const int rw = rq_data_dir(req);
struct hd_struct *part = get_part(disk, req->sector);
- __all_stat_inc(disk, ios[rw], req->sector);
- __all_stat_add(disk, ticks[rw], duration, req->sector);
+ __all_stat_inc(disk, part, ios[rw], req->sector);
+ __all_stat_add(disk, part, ticks[rw], duration, req->sector);
disk_round_stats(disk);
disk->in_flight--;
if (part) {
diff --git a/drivers/block/aoe/aoecmd.c b/drivers/block/aoe/aoecmd.c
index 8fc429c..41f818b 100644
--- a/drivers/block/aoe/aoecmd.c
+++ b/drivers/block/aoe/aoecmd.c
@@ -755,11 +755,13 @@
{
unsigned long n_sect = bio->bi_size >> 9;
const int rw = bio_data_dir(bio);
+ struct hd_struct *part;
- all_stat_inc(disk, ios[rw], sector);
- all_stat_add(disk, ticks[rw], duration, sector);
- all_stat_add(disk, sectors[rw], n_sect, sector);
- all_stat_add(disk, io_ticks, duration, sector);
+ part = get_part(disk, sector);
+ all_stat_inc(disk, part, ios[rw], sector);
+ all_stat_add(disk, part, ticks[rw], duration, sector);
+ all_stat_add(disk, part, sectors[rw], n_sect, sector);
+ all_stat_add(disk, part, io_ticks, duration, sector);
}
void
diff --git a/include/linux/genhd.h b/include/linux/genhd.h
index ecd2bf6..e9874e7 100644
--- a/include/linux/genhd.h
+++ b/include/linux/genhd.h
@@ -178,17 +178,17 @@
static inline void disk_stat_set_all(struct gendisk *gendiskp, int value) {
int i;
+
for_each_possible_cpu(i)
memset(per_cpu_ptr(gendiskp->dkstats, i), value,
- sizeof (struct disk_stats));
+ sizeof(struct disk_stats));
}
#define __part_stat_add(part, field, addnd) \
(per_cpu_ptr(part->dkstats, smp_processor_id())->field += addnd)
-#define __all_stat_add(gendiskp, field, addnd, sector) \
+#define __all_stat_add(gendiskp, part, field, addnd, sector) \
({ \
- struct hd_struct *part = get_part(gendiskp, sector); \
if (part) \
__part_stat_add(part, field, addnd); \
__disk_stat_add(gendiskp, field, addnd); \
@@ -203,11 +203,13 @@
res; \
})
-static inline void part_stat_set_all(struct hd_struct *part, int value) {
+static inline void part_stat_set_all(struct hd_struct *part, int value)
+{
int i;
+
for_each_possible_cpu(i)
memset(per_cpu_ptr(part->dkstats, i), value,
- sizeof(struct disk_stats));
+ sizeof(struct disk_stats));
}
#else /* !CONFIG_SMP */
@@ -223,9 +225,8 @@
#define __part_stat_add(part, field, addnd) \
(part->dkstats.field += addnd)
-#define __all_stat_add(gendiskp, field, addnd, sector) \
+#define __all_stat_add(gendiskp, part, field, addnd, sector) \
({ \
- struct hd_struct *part = get_part(gendiskp, sector); \
if (part) \
part->dkstats.field += addnd; \
__disk_stat_add(gendiskp, field, addnd); \
@@ -276,10 +277,10 @@
#define part_stat_sub(gendiskp, field, subnd) \
part_stat_add(gendiskp, field, -subnd)
-#define all_stat_add(gendiskp, field, addnd, sector) \
+#define all_stat_add(gendiskp, part, field, addnd, sector) \
do { \
preempt_disable(); \
- __all_stat_add(gendiskp, field, addnd, sector); \
+ __all_stat_add(gendiskp, part, field, addnd, sector); \
preempt_enable(); \
} while (0)
@@ -288,15 +289,15 @@
#define all_stat_dec(gendiskp, field, sector) \
all_stat_add(gendiskp, field, -1, sector)
-#define __all_stat_inc(gendiskp, field, sector) \
- __all_stat_add(gendiskp, field, 1, sector)
-#define all_stat_inc(gendiskp, field, sector) \
- all_stat_add(gendiskp, field, 1, sector)
+#define __all_stat_inc(gendiskp, part, field, sector) \
+ __all_stat_add(gendiskp, part, field, 1, sector)
+#define all_stat_inc(gendiskp, part, field, sector) \
+ all_stat_add(gendiskp, part, field, 1, sector)
-#define __all_stat_sub(gendiskp, field, subnd, sector) \
- __all_stat_add(gendiskp, field, -subnd, sector)
-#define all_stat_sub(gendiskp, field, subnd, sector) \
- all_stat_add(gendiskp, field, -subnd, sector)
+#define __all_stat_sub(gendiskp, part, field, subnd, sector) \
+ __all_stat_add(gendiskp, part, field, -subnd, sector)
+#define all_stat_sub(gendiskp, part, field, subnd, sector) \
+ all_stat_add(gendiskp, part, field, -subnd, sector)
/* Inlines to alloc and free disk stats in struct gendisk */
#ifdef CONFIG_SMP