BACKPORT: block: annotate refault stalls from IO submission
psi tracks the time tasks wait for refaulting pages to become
uptodate, but it does not track the time spent submitting the IO. The
submission part can be significant if backing storage is contended or
when cgroup throttling (io.latency) is in effect - a lot of time is
spent in submit_bio(). In that case, we underreport memory pressure.
Annotate submit_bio() to account submission time as memory stall when
the bio is reading userspace workingset pages.
Tested-by: Suren Baghdasaryan <surenb@google.com>
Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
(cherry picked from commit b8e24a9300b0836a9d39f6b20746766b3b81f1bd)
Conflicts:
include/linux/blk_types.h
block/bio.c
(1. Manually resolved BIO_WORKINGSET being definition instead of enum.
2. Add bio_set_flag call into bio_add_page instead of not yet existing
__bio_add_page)
Bug: 141131229
Test: boot and run act test suite
Change-Id: I99cef039844e219f1dc8196feead54b6f5fb26bb
Signed-off-by: Suren Baghdasaryan <surenb@google.com>
diff --git a/block/bio.c b/block/bio.c
index 4c18a68..0c03c47 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -849,6 +849,9 @@
bio->bi_vcnt++;
done:
bio->bi_iter.bi_size += len;
+
+ if (!bio_flagged(bio, BIO_WORKINGSET) && unlikely(PageWorkingset(page)))
+ bio_set_flag(bio, BIO_WORKINGSET);
return len;
}
EXPORT_SYMBOL(bio_add_page);
diff --git a/block/blk-core.c b/block/blk-core.c
index 0ce29cd..49951cd 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -33,6 +33,7 @@
#include <linux/ratelimit.h>
#include <linux/pm_runtime.h>
#include <linux/blk-cgroup.h>
+#include <linux/psi.h>
#define CREATE_TRACE_POINTS
#include <trace/events/block.h>
@@ -2091,6 +2092,10 @@
*/
blk_qc_t submit_bio(struct bio *bio)
{
+ bool workingset_read = false;
+ unsigned long pflags;
+ blk_qc_t ret;
+
/*
* If it's a regular read/write or a barrier with data attached,
* go through the normal accounting stuff before submission.
@@ -2106,6 +2111,8 @@
if (op_is_write(bio_op(bio))) {
count_vm_events(PGPGOUT, count);
} else {
+ if (bio_flagged(bio, BIO_WORKINGSET))
+ workingset_read = true;
task_io_account_read(bio->bi_iter.bi_size);
count_vm_events(PGPGIN, count);
}
@@ -2121,7 +2128,21 @@
}
}
- return generic_make_request(bio);
+ /*
+ * If we're reading data that is part of the userspace
+ * workingset, count submission time as memory stall. When the
+ * device is congested, or the submitting cgroup IO-throttled,
+ * submission can be a significant part of overall IO time.
+ */
+ if (workingset_read)
+ psi_memstall_enter(&pflags);
+
+ ret = generic_make_request(bio);
+
+ if (workingset_read)
+ psi_memstall_leave(&pflags);
+
+ return ret;
}
EXPORT_SYMBOL(submit_bio);
diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index 99ad610..26b3fd3 100644
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -117,9 +117,10 @@
#define BIO_BOUNCED 3 /* bio is a bounce bio */
#define BIO_USER_MAPPED 4 /* contains user pages */
#define BIO_NULL_MAPPED 5 /* contains invalid user pages */
-#define BIO_QUIET 6 /* Make BIO Quiet */
-#define BIO_CHAIN 7 /* chained bio, ->bi_remaining in effect */
-#define BIO_REFFED 8 /* bio has elevated ->bi_cnt */
+#define BIO_WORKINGSET 6 /* contains userspace workingset pages */
+#define BIO_QUIET 7 /* Make BIO Quiet */
+#define BIO_CHAIN 8 /* chained bio, ->bi_remaining in effect */
+#define BIO_REFFED 9 /* bio has elevated ->bi_cnt */
/*
* Flags starting here get preserved by bio_reset() - this includes