Syed Rameez Mustafa | dddcab7 | 2016-09-07 16:18:27 -0700 | [diff] [blame^] | 1 | /* Copyright (c) 2012, 2015, The Linux Foundation. All rights reserved. |
| 2 | * |
| 3 | * This program is free software; you can redistribute it and/or modify |
| 4 | * it under the terms of the GNU General Public License version 2 and |
| 5 | * only version 2 as published by the Free Software Foundation. |
| 6 | * |
| 7 | * This program is distributed in the hope that it will be useful, |
| 8 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 9 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 10 | * GNU General Public License for more details. |
| 11 | */ |
| 12 | |
| 13 | /* |
| 14 | * Scheduler hook for average runqueue determination |
| 15 | */ |
| 16 | #include <linux/module.h> |
| 17 | #include <linux/percpu.h> |
| 18 | #include <linux/hrtimer.h> |
| 19 | #include <linux/sched.h> |
| 20 | #include <linux/math64.h> |
| 21 | |
| 22 | #include "sched.h" |
| 23 | #include <trace/events/sched.h> |
| 24 | |
| 25 | static DEFINE_PER_CPU(u64, nr_prod_sum); |
| 26 | static DEFINE_PER_CPU(u64, last_time); |
| 27 | static DEFINE_PER_CPU(u64, nr_big_prod_sum); |
| 28 | static DEFINE_PER_CPU(u64, nr); |
| 29 | |
| 30 | static DEFINE_PER_CPU(unsigned long, iowait_prod_sum); |
| 31 | static DEFINE_PER_CPU(spinlock_t, nr_lock) = __SPIN_LOCK_UNLOCKED(nr_lock); |
| 32 | static s64 last_get_time; |
| 33 | |
| 34 | /** |
| 35 | * sched_get_nr_running_avg |
| 36 | * @return: Average nr_running, iowait and nr_big_tasks value since last poll. |
| 37 | * Returns the avg * 100 to return up to two decimal points |
| 38 | * of accuracy. |
| 39 | * |
| 40 | * Obtains the average nr_running value since the last poll. |
| 41 | * This function may not be called concurrently with itself |
| 42 | */ |
| 43 | void sched_get_nr_running_avg(int *avg, int *iowait_avg, int *big_avg) |
| 44 | { |
| 45 | int cpu; |
| 46 | u64 curr_time = sched_clock(); |
| 47 | u64 diff = curr_time - last_get_time; |
| 48 | u64 tmp_avg = 0, tmp_iowait = 0, tmp_big_avg = 0; |
| 49 | |
| 50 | *avg = 0; |
| 51 | *iowait_avg = 0; |
| 52 | *big_avg = 0; |
| 53 | |
| 54 | if (!diff) |
| 55 | return; |
| 56 | |
| 57 | /* read and reset nr_running counts */ |
| 58 | for_each_possible_cpu(cpu) { |
| 59 | unsigned long flags; |
| 60 | |
| 61 | spin_lock_irqsave(&per_cpu(nr_lock, cpu), flags); |
| 62 | curr_time = sched_clock(); |
| 63 | tmp_avg += per_cpu(nr_prod_sum, cpu); |
| 64 | tmp_avg += per_cpu(nr, cpu) * |
| 65 | (curr_time - per_cpu(last_time, cpu)); |
| 66 | |
| 67 | tmp_big_avg += per_cpu(nr_big_prod_sum, cpu); |
| 68 | tmp_big_avg += nr_eligible_big_tasks(cpu) * |
| 69 | (curr_time - per_cpu(last_time, cpu)); |
| 70 | |
| 71 | tmp_iowait += per_cpu(iowait_prod_sum, cpu); |
| 72 | tmp_iowait += nr_iowait_cpu(cpu) * |
| 73 | (curr_time - per_cpu(last_time, cpu)); |
| 74 | |
| 75 | per_cpu(last_time, cpu) = curr_time; |
| 76 | |
| 77 | per_cpu(nr_prod_sum, cpu) = 0; |
| 78 | per_cpu(nr_big_prod_sum, cpu) = 0; |
| 79 | per_cpu(iowait_prod_sum, cpu) = 0; |
| 80 | |
| 81 | spin_unlock_irqrestore(&per_cpu(nr_lock, cpu), flags); |
| 82 | } |
| 83 | |
| 84 | diff = curr_time - last_get_time; |
| 85 | last_get_time = curr_time; |
| 86 | |
| 87 | *avg = (int)div64_u64(tmp_avg * 100, diff); |
| 88 | *big_avg = (int)div64_u64(tmp_big_avg * 100, diff); |
| 89 | *iowait_avg = (int)div64_u64(tmp_iowait * 100, diff); |
| 90 | |
| 91 | trace_sched_get_nr_running_avg(*avg, *big_avg, *iowait_avg); |
| 92 | |
| 93 | BUG_ON(*avg < 0 || *big_avg < 0 || *iowait_avg < 0); |
| 94 | pr_debug("%s - avg:%d big_avg:%d iowait_avg:%d\n", |
| 95 | __func__, *avg, *big_avg, *iowait_avg); |
| 96 | } |
| 97 | EXPORT_SYMBOL(sched_get_nr_running_avg); |
| 98 | |
| 99 | /** |
| 100 | * sched_update_nr_prod |
| 101 | * @cpu: The core id of the nr running driver. |
| 102 | * @delta: Adjust nr by 'delta' amount |
| 103 | * @inc: Whether we are increasing or decreasing the count |
| 104 | * @return: N/A |
| 105 | * |
| 106 | * Update average with latest nr_running value for CPU |
| 107 | */ |
| 108 | void sched_update_nr_prod(int cpu, long delta, bool inc) |
| 109 | { |
| 110 | int diff; |
| 111 | s64 curr_time; |
| 112 | unsigned long flags, nr_running; |
| 113 | |
| 114 | spin_lock_irqsave(&per_cpu(nr_lock, cpu), flags); |
| 115 | nr_running = per_cpu(nr, cpu); |
| 116 | curr_time = sched_clock(); |
| 117 | diff = curr_time - per_cpu(last_time, cpu); |
| 118 | per_cpu(last_time, cpu) = curr_time; |
| 119 | per_cpu(nr, cpu) = nr_running + (inc ? delta : -delta); |
| 120 | |
| 121 | BUG_ON((s64)per_cpu(nr, cpu) < 0); |
| 122 | |
| 123 | per_cpu(nr_prod_sum, cpu) += nr_running * diff; |
| 124 | per_cpu(nr_big_prod_sum, cpu) += nr_eligible_big_tasks(cpu) * diff; |
| 125 | per_cpu(iowait_prod_sum, cpu) += nr_iowait_cpu(cpu) * diff; |
| 126 | spin_unlock_irqrestore(&per_cpu(nr_lock, cpu), flags); |
| 127 | } |
| 128 | EXPORT_SYMBOL(sched_update_nr_prod); |